Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): implement the Lambda Extensions API (#1773)
Jay9f0dec9317d2
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.javaindex 89bda916ff..0996536ff3 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java@@ -57,22 +57,53 @@ public class ContainerLogStreamer {.withStdErr(true).withFollowStream(true).withTimestamps(false)- .exec(new ResultCallback.Adapter<>() {- @Override- public void onNext(Frame frame) {- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();- if (!line.isEmpty()) {- LOG.infov("[{0}] {1}", logPrefix, line);- forwardToCloudWatchLogs(logGroup, logStream, region, line);- }- }- });+ .exec(frameCallback(logGroup, logStream, region, logPrefix));} catch (Exception e) {LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());return null;}}+ /**+ * Streams the output of a {@code docker exec} to the same destinations as {@link #attach}.+ *+ * <p>Needed because {@code docker logs} (and therefore {@link #attach}, which uses+ * {@code logContainerCmd}) only covers the container's PID 1 stdout/stderr. An exec runs on a+ * separate Docker stream that never reaches that log, so a Lambda extension started via exec+ * would otherwise produce no CloudWatch output at all — the opposite of what an observability+ * extension is for.+ *+ * <p>The returned callback must be handed to {@code execStartCmd(...).exec(...)} by the caller,+ * which owns the exec's lifecycle. Draining it is required regardless of logging: an undrained+ * exec output pipe fills up and stalls the process on the other end.+ *+ * @param logPrefix console-only prefix (e.g. {@code "lambda:myFn:my-extension"}); the message+ * forwarded to CloudWatch is left unprefixed so the log stream matches real+ * AWS, which interleaves extension output with the function's own.+ */+ public ResultCallback.Adapter<Frame> execLogCallback(String logGroup, String logStream,+ String region, String logPrefix) {+ return frameCallback(logGroup, logStream, region, logPrefix);+ }++ /** Shared frame handling for both the container log stream and exec streams. */+ private ResultCallback.Adapter<Frame> frameCallback(String logGroup, String logStream,+ String region, String logPrefix) {+ return new ResultCallback.Adapter<>() {+ @Override+ public void onNext(Frame frame) {+ if (frame == null || frame.getPayload() == null) {+ return;+ }+ String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();+ if (!line.isEmpty()) {+ LOG.infov("[{0}] {1}", logPrefix, line);+ forwardToCloudWatchLogs(logGroup, logStream, region, line);+ }+ }+ };+ }+/*** Creates a CloudWatch log group and stream if they don't already exist.*/diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.javaindex e6b4361d9a..b81829a83e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java@@ -117,6 +117,15 @@ public class WarmPool implements ContainerTeardown {if (candidate == null) {break;}+ // A container whose extension reported a fatal error is still *running*, so the+ // liveness probe alone would hand it back out. Skip it for the same reason a dead+ // one is skipped: it can no longer serve invocations correctly.+ if (candidate.isFaulted()) {+ LOG.infov("Discarding pooled container {0} for function {1}: an extension reported a fatal error",+ candidate.getContainerId(), fn.getFunctionName());+ stopQuietly(candidate);+ continue;+ }if (containerLauncher.isAlive(candidate)) {handle = candidate;break;@@ -145,6 +154,16 @@ public class WarmPool implements ContainerTeardown {*/public void release(ContainerHandle handle) {boolean ephemeral = config != null && config.services().lambda().ephemeral();+ // An extension reporting an init/exit error is fatal to the execution environment in real+ // AWS. RuntimeApiServer already refuses new work at that point; the container is torn down+ // here rather than at fault time so the invocation that was in flight when the extension+ // failed still completes normally through the runtime.+ if (handle.isFaulted()) {+ LOG.infov("Retiring container {0} for function {1}: an extension reported a fatal error",+ handle.getContainerId(), handle.getFunctionName());+ stopQuietly(handle);+ return;+ }if (ephemeral || handle.isHotReload()) {LOG.debugv("{0}: stopping container {1} after invocation",handle.isHotReload() ? "Hot-reload" : "Ephemeral", handle.getContainerId());diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.javaindex abd458d7f7..7e78bd1487 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java@@ -44,6 +44,18 @@ public class ContainerHandle {public void touchLastUsed() { this.lastUsedMs = System.currentTimeMillis(); }public ContainerState getState() { return state; }public void setState(ContainerState state) { this.state = state; }+public Closeable getLogStream() { return logStream; }public void setLogStream(Closeable logStream) { this.logStream = logStream; }++ /**+ * True once an extension in this container reported an init or exit error. Real AWS treats+ * both as fatal to the execution environment, so {@code WarmPool} must retire the container+ * instead of pooling or reusing it. Read from the container's {@code RuntimeApiServer}, which+ * is where extensions report the error; a handle with no runtime server (test fixtures) is+ * never faulted.+ */+ public boolean isFaulted() {+ return runtimeApiServer != null && runtimeApiServer.isFaulted();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 1f4e1cc99d..0925c9d04f 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -17,11 +17,13 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.exception.NotFoundException;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import java.io.Closeable;@@ -147,6 +149,8 @@ public class ContainerLauncher {// Start Runtime API server first so container can connect on bootRuntimeApiServer runtimeApiServer = runtimeApiServerFactory.create();+ runtimeApiServer.setFunctionMetadata(fn.getFunctionName(), fn.getVersion(), fn.getHandler(),+ fn.getAccountId());// Everything after the runtime-api server is allocated runs inside one try/catch: a failure// ANYWHERE below — image/host resolve, the code-volume populate (ensureCodeVolume), the spec@@ -328,6 +332,29 @@ public class ContainerLauncher {// Now start the container with code in placelifecycleManager.startCreated(containerId, spec);+ // Extensions can log as soon as they start, which is before the container's own log stream+ // is attached below. Create the group/stream up front so those early lines are not dropped+ // by CloudWatch; the call is idempotent, so attach() repeating it is harmless.+ LogDestination logDestination = new LogDestination(cwLogGroup, cwLogStream, lambdaRegion);+ logStreamer.ensureLogGroupAndStream(cwLogGroup, cwLogStream, lambdaRegion);++ // Real AWS's runtime interface client discovers and launches every binary under+ // /opt/extensions/ as a sibling process to the main entrypoint before the runtime is+ // considered ready; Floci only runs the image's own ENTRYPOINT/CMD, so extensions+ // (e.g. aws-lambda-web-adapter) never start without this. Best-effort: an extension+ // launch failure shouldn't fail the whole container launch, since a function with no+ // extensions is the common case and this must be a no-op for it.+ launchExtensions(dockerClient, containerId, fn.getFunctionName(), runtimeApiServer, logDestination);++ // Init-readiness barrier: the execs above are detached, so without waiting here the caller+ // could enqueue the first invocation before an extension is ready to receive it — the+ // adapter would silently miss that invoke. Real AWS likewise holds the environment out of+ // service until every extension is init-ready, which it defines as the extension's first+ // /extension/event/next rather than its register call. A no-extensions function (the+ // common case) does not wait at all; a timeout is non-fatal, since a container that serves+ // invocations without a slow extension is strictly better than failing the launch.+ awaitExtensionReadiness(runtimeApiServer, fn.getFunctionName());+ContainerHandle handle = new ContainerHandle(containerId, fn.getFunctionName(), runtimeApiServer, ContainerState.WARM, fn.isHotReload());// Attach log streaming@@ -600,6 +627,136 @@ public class ContainerLauncher {return runtime != null && runtime.startsWith("provided");}+ /** Directory Lambda's real init system scans for extension binaries. */+ private static final String EXTENSIONS_DIR = "/opt/extensions";++ /**+ * How long a launch waits for extensions to call {@code /extension/register} before giving up+ * and serving invocations without them. Real AWS allows extensions the function's full init+ * budget (10s); this is deliberately shorter, since exceeding it here is non-fatal and the+ * cost of the wait is paid on every cold start of a function that has extensions.+ */+ private static final long EXTENSION_REGISTRATION_TIMEOUT_MS = 5000;++ /**+ * Discovers binaries under {@link #EXTENSIONS_DIR} inside the (already started) container and+ * launches each as a detached process — the piece of real AWS's init system (which starts every+ * registered extension alongside the runtime) that Floci otherwise has no equivalent for. Uses+ * {@code docker exec} rather than baking a wrapper entrypoint into the image, since the image is+ * user-supplied and unmodified.+ *+ * <p>Extensions inherit the container's env (already set on the container itself, including+ * {@code AWS_LAMBDA_RUNTIME_API}) automatically — {@code docker exec} does not need it re-supplied.+ * Best-effort throughout: a function with no {@code /opt/extensions} directory (the common case)+ * must be a silent no-op, and a failure launching one extension must not prevent the container+ * (or other extensions) from running.+ */+ /** Where a container's output is sent: the CloudWatch log group/stream and its region. */+ private record LogDestination(String logGroup, String logStream, String region) { }++ private void launchExtensions(DockerClient dockerClient, String containerId, String functionName,+ RuntimeApiServer runtimeApiServer, LogDestination logDestination) {+ List<String> extensionNames = listExtensionBinaries(dockerClient, containerId, functionName);+ // Arm the readiness barrier before starting any extension process, so the latch already+ // exists when the first one starts polling /extension/event/next — otherwise a+ // fast-starting extension could become ready before there is anything to count it down.+ runtimeApiServer.expectExtensions(extensionNames.size());+ for (String name : extensionNames) {+ try {+ String path = EXTENSIONS_DIR + "/" + name;+ var create = dockerClient.execCreateCmd(containerId)+ .withCmd(path)+ .withAttachStdout(true)+ .withAttachStderr(true);+ String execId = create.exec().getId();+ // Detached: an extension runs for the container's whole lifetime, so this must not+ // block the launch waiting for it to exit.+ //+ // The callback forwards the exec's output to the function's CloudWatch log group,+ // the same destination the container's PID 1 output goes to. That has to be done+ // explicitly: `docker logs` only covers PID 1, so an exec's stream never reaches+ // the container log and an observability extension's output would vanish entirely.+ // Draining is also required in its own right — an unread exec pipe fills up and+ // stalls the extension process.+ dockerClient.execStartCmd(execId).exec(logStreamer.execLogCallback(+ logDestination.logGroup(), logDestination.logStream(), logDestination.region(),+ "lambda:" + functionName + ":" + name));+ LOG.infov("Launched extension {0} for function {1} (container {2})",+ name, functionName, containerId);+ } catch (Exception e) {+ LOG.warnv(e, "Failed to launch extension {0} for function {1}; continuing without it",+ name, functionName);+ }+ }+ }++ /**+ * Waits for every launched extension to become init-ready — its first+ * {@code /extension/event/next}, the point at which AWS considers an extension initialised —+ * before the container is handed to the caller for its first invocation.+ *+ * <p>Best-effort by design: a timeout logs and proceeds rather than failing the launch. An+ * extension that is slow or crashes on startup should degrade to "invocations run without it"+ * — the same outcome as before this barrier existed — not take the whole function down. A+ * genuinely broken extension reports {@code init/error} instead, which condemns the+ * environment through {@code RuntimeApiServer}'s fault path.+ */+ private void awaitExtensionReadiness(RuntimeApiServer runtimeApiServer, String functionName) {+ try {+ if (!runtimeApiServer.awaitExtensionsReady(EXTENSION_REGISTRATION_TIMEOUT_MS)) {+ LOG.warnv("Not all extensions for function {0} became ready within {1}ms; "+ + "continuing without them",+ functionName, String.valueOf(EXTENSION_REGISTRATION_TIMEOUT_MS));+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ LOG.debugv("Interrupted waiting for extensions of function {0} to become ready", functionName);+ }+ }++ /**+ * Lists file names directly under {@link #EXTENSIONS_DIR} inside the container, or an empty+ * list if the directory doesn't exist or the probe fails (most functions have no extensions).+ *+ * <p>Uses Docker's archive API ({@code copyArchiveFromContainerCmd}) rather than {@code exec}ing+ * a shell/{@code find}/{@code basename} pipeline inside the container: a minimal or distroless+ * function image can have a valid extension binary under {@code /opt/extensions} without any of+ * those utilities present, which would silently look like "no extensions" to a shell-based probe.+ * The archive API only talks to the Docker daemon, so it works regardless of the image contents.+ */+ private List<String> listExtensionBinaries(DockerClient dockerClient, String containerId, String functionName) {+ try (InputStream tarStream = dockerClient.copyArchiveFromContainerCmd(containerId, EXTENSIONS_DIR).exec();+ TarArchiveInputStream tar = new TarArchiveInputStream(tarStream)) {++ List<String> names = new ArrayList<>();+ TarArchiveEntry entry;+ while ((entry = tar.getNextEntry()) != null) {+ if (!tar.canReadEntryData(entry) || entry.isDirectory()) {+ continue;+ }+ // Docker wraps the requested directory itself as a leading path component (e.g.+ // "extensions/lambda-adapter"); only direct children count as extension binaries,+ // matching the real init system's non-recursive scan of the directory.+ String name = entry.getName();+ int slash = name.indexOf('/');+ if (slash < 0 || name.indexOf('/', slash + 1) >= 0) {+ continue;+ }+ if ((entry.getMode() & 0111) == 0) {+ continue;+ }+ names.add(name.substring(slash + 1));+ }+ return names;+ } catch (NotFoundException e) {+ return List.of();+ } catch (Exception e) {+ LOG.debugv("Could not list {0} for function {1} ({2}); assuming no extensions",+ EXTENSIONS_DIR, functionName, e.getMessage());+ return List.of();+ }+ }+/*** Resolves the host path of Floci's CA certificate to inject into Lambda containers, or* empty when TLS is disabled or no readable certificate exists. Mirrors {@code TlsConfigSource}:diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.javanew file mode 100644index 0000000000..ed851f6c78--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.java@@ -0,0 +1,39 @@+package io.github.hectorvent.floci.services.lambda.model;++/**+ * A lifecycle event delivered to a registered extension via+ * {@code GET /2020-01-01/extension/event/next} — either {@code INVOKE} (mirrors a runtime+ * invocation) or {@code SHUTDOWN} (container is about to stop).+ */+public class ExtensionEvent {++ public enum Type { INVOKE, SHUTDOWN }++ private final Type type;+ private final String requestId;+ private final long deadlineMs;+ private final String functionArn;+ private final String shutdownReason;++ public static ExtensionEvent invoke(String requestId, long deadlineMs, String functionArn) {+ return new ExtensionEvent(Type.INVOKE, requestId, deadlineMs, functionArn, null);+ }++ public static ExtensionEvent shutdown(long deadlineMs, String shutdownReason) {+ return new ExtensionEvent(Type.SHUTDOWN, null, deadlineMs, null, shutdownReason);+ }++ private ExtensionEvent(Type type, String requestId, long deadlineMs, String functionArn, String shutdownReason) {+ this.type = type;+ this.requestId = requestId;+ this.deadlineMs = deadlineMs;+ this.functionArn = functionArn;+ this.shutdownReason = shutdownReason;+ }++ public Type getType() { return type; }+ public String getRequestId() { return requestId; }+ public long getDeadlineMs() { return deadlineMs; }+ public String getFunctionArn() { return functionArn; }+ public String getShutdownReason() { return shutdownReason; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.javanew file mode 100644index 0000000000..6eb5feda09--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.java@@ -0,0 +1,67 @@+package io.github.hectorvent.floci.services.lambda.model;++import java.util.ArrayDeque;+import java.util.List;+import io.vertx.ext.web.RoutingContext;++/**+ * Tracks one extension's Extensions API registration state within a single container's+ * {@code RuntimeApiServer} — its subscribed event types and the queue/parked-poller pair+ * used to deliver {@link ExtensionEvent}s to its {@code /extension/event/next} polling loop.+ *+ * Not thread-safe on its own — all access to {@code pendingEvents}/{@code waitingContext}+ * must happen while holding the owning {@code RuntimeApiServer}'s lock. This class+ * intentionally has no internal synchronization; see {@code RuntimeApiServer}'s class doc+ * for the locking discipline.+ */+public class RegisteredExtension {++ private final String identifier;+ private final String name;+ private final List<String> subscribedEvents;+ private final ArrayDeque<ExtensionEvent> pendingEvents = new ArrayDeque<>();+ private RoutingContext waitingContext;+ private boolean firstNextReceived;++ public RegisteredExtension(String identifier, String name, List<String> subscribedEvents) {+ this.identifier = identifier;+ this.name = name;+ this.subscribedEvents = subscribedEvents;+ }++ public String getIdentifier() { return identifier; }+ public String getName() { return name; }+ public List<String> getSubscribedEvents() { return subscribedEvents; }++ public boolean isSubscribedTo(ExtensionEvent.Type type) {+ return subscribedEvents.contains(type.name());+ }++ public ArrayDeque<ExtensionEvent> getPendingEvents() { return pendingEvents; }++ public RoutingContext takeWaitingContext() {+ RoutingContext ctx = waitingContext;+ waitingContext = null;+ return ctx;+ }++ public void setWaitingContext(RoutingContext ctx) {+ waitingContext = ctx;+ }++ /**+ * Records that this extension has issued its first {@code /extension/event/next}, which is+ * what AWS treats as the extension being init-ready (registering alone only obtains an+ * identifier). Called under the owning server's lock, like the rest of this class.+ *+ * @return true only on the first call, so the caller counts the readiness barrier down once+ * per extension no matter how many times it polls.+ */+ public boolean markFirstNextReceived() {+ if (firstNextReceived) {+ return false;+ }+ firstNextReceived = true;+ return true;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.javaindex 9e5d2095ea..8dd861a27d 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java@@ -1,27 +1,67 @@package io.github.hectorvent.floci.services.lambda.runtime;+import io.github.hectorvent.floci.services.lambda.model.ExtensionEvent;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;+import io.github.hectorvent.floci.services.lambda.model.RegisteredExtension;import io.vertx.core.Vertx;import io.vertx.core.http.HttpServer;import io.vertx.core.http.HttpServerOptions;+import io.vertx.core.json.JsonObject;import io.vertx.ext.web.Router;import io.vertx.ext.web.RoutingContext;import io.vertx.ext.web.handler.BodyHandler;import org.jboss.logging.Logger;+import java.util.ArrayDeque;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.UUID;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ConcurrentLinkedQueue;+import java.util.concurrent.CountDownLatch;/**- * Per-container HTTP server implementing the AWS Lambda Runtime API.+ * Per-container HTTP server implementing the AWS Lambda Runtime API and, for+ * Lambda Extensions (e.g. aws-lambda-web-adapter), the Extensions API.* NOT a CDI bean — instances are created by RuntimeApiServerFactory.** The container's language runtime connects to this server to:* - Poll for the next invocation (GET /runtime/invocation/next)* - Report success (POST /runtime/invocation/{requestId}/response)* - Report failure (POST /runtime/invocation/{requestId}/error)+ *+ * Extension processes (binaries under /opt/extensions/) connect to this server to:+ * - Register for lifecycle events (POST /extension/register)+ * - Poll for the next event (GET /extension/event/next)+ * - Report an init/exit error (POST /extension/init/error, /extension/exit/error)+ *+ * Extension event delivery is intentionally decoupled from invoke completion: unlike real+ * AWS (where the Invoke phase only ends once the runtime AND every registered extension have+ * each signaled done via /next), completing a PendingInvocation's resultFuture here depends+ * only on the runtime's own /response or /error call. Extensions are still notified of every+ * INVOKE/SHUTDOWN via their own /event/next polling loop. This is a deliberate simplification:+ * it's enough for extensions like aws-lambda-web-adapter (whose registration alone is what+ * gates it starting its proxy loop) without taking on exact post-invoke completion timing,+ * which mainly matters for extensions doing work after the response is already returned to+ * the client (e.g. flushing telemetry) — not a fidelity floci's local dev/test use case needs.+ *+ * Locking discipline: every read/write of {@code stopped}, {@code pendingQueue},+ * {@code waitingContexts}, {@code extensions}, and each {@code RegisteredExtension}'s+ * {@code pendingEvents}/waiting context happens inside a single {@code synchronized(lock)}+ * block per operation. That block only *decides* what to do (dispatch to a specific+ * RoutingContext, complete a future, or nothing) and returns/collects that decision — the+ * actual {@code ctx.response()...end()}/{@code vertx.runOnContext(...)} calls always happen+ * after releasing the lock, so the lock is never held across a call into Vert.x and hold+ * times stay to sub-microsecond in-memory queue operations. One lock per server instance+ * (up to ~100 instances live at once, one per container) means no cross-container contention.+ * This intentionally stays non-blocking/event-loop-based throughout — NEXT_PATH and+ * EXTENSION_NEXT_PATH never park a real thread in {@code wait()} — because the container's+ * language-runtime bootstrap process holds one of these long-polls open for its entire idle+ * lifetime; blocking a bounded worker-thread pool (Quarkus's default is 20 threads) for that+ * long would exhaust it with only a handful of warm containers.*/public class RuntimeApiServer {@@ -33,6 +73,26 @@ public class RuntimeApiServer {private static final String ERROR_PATH = "/" + RUNTIME_API_VERSION + "/runtime/invocation/:requestId/error";private static final String INIT_ERROR_PATH = "/" + RUNTIME_API_VERSION + "/runtime/init/error";+ private static final String EXTENSIONS_API_VERSION = "2020-01-01";+ private static final String EXTENSION_REGISTER_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/register";+ private static final String EXTENSION_NEXT_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/event/next";+ private static final String EXTENSION_INIT_ERROR_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/init/error";+ private static final String EXTENSION_EXIT_ERROR_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/exit/error";++ private static final String EXTENSION_NAME_HEADER = "Lambda-Extension-Name";+ private static final String EXTENSION_ID_HEADER = "Lambda-Extension-Identifier";+ private static final String EXTENSION_EVENT_ID_HEADER = "Lambda-Extension-Event-Identifier";+ private static final String EXTENSION_ACCEPT_FEATURE_HEADER = "Lambda-Extension-Accept-Feature";++ /** Required by AWS on both /extension/init/error and /extension/exit/error. */+ private static final String EXTENSION_ERROR_TYPE_HEADER = "Lambda-Extension-Function-Error-Type";++ /** Opt-in feature that adds {@code accountId} to the register response. */+ private static final String ACCOUNT_ID_FEATURE = "accountId";++ /** Matches the account used elsewhere when a function carries none (see LambdaService). */+ private static final String DEFAULT_ACCOUNT_ID = "000000000000";+private static final byte[] CONTAINER_STOPPED_PAYLOAD ="{\"errorMessage\":\"Container stopped\",\"errorType\":\"ContainerStopped\"}".getBytes();@@ -44,27 +104,120 @@ public class RuntimeApiServer {private final Vertx vertx;private final int port;- // Invocations queued before a /next poller arrived.- private final ConcurrentLinkedQueue<PendingInvocation> pendingQueue = new ConcurrentLinkedQueue<>();+ // Guards stopped, pendingQueue, waitingContexts, extensions, closeFuture, and every+ // RegisteredExtension's pendingEvents/waitingContext. See class doc for the discipline.+ private final Object lock = new Object();- // /next callers parked while the pending queue is empty.- private final ConcurrentLinkedQueue<RoutingContext> waitingContexts = new ConcurrentLinkedQueue<>();+ // Invocations queued before a /next poller arrived. Guarded by lock.+ private final ArrayDeque<PendingInvocation> pendingQueue = new ArrayDeque<>();++ // /next callers parked while the pending queue is empty. Guarded by lock.+ private final ArrayDeque<RoutingContext> waitingContexts = new ArrayDeque<>();private final ConcurrentHashMap<String, PendingInvocation> inFlight = new ConcurrentHashMap<>();+ // Extensions registered via /extension/register, keyed by their generated identifier.+ // Guarded by lock.+ private final Map<String, RegisteredExtension> extensions = new HashMap<>();+private volatile HttpServer httpServer;- private volatile boolean stopped;- private volatile CompletableFuture<Void> closeFuture;+ private boolean stopped;+ private CompletableFuture<Void> closeFuture;++ // Set once an extension reports an init/exit error. Real AWS treats both as fatal to the+ // execution environment, so the container must neither accept new work nor be reused.+ //+ // Read under `lock` alongside `stopped` in enqueue()/NEXT_PATH — a condemned environment+ // refuses new invocations rather than queueing them to a runtime that will never receive+ // them — and read without the lock by WarmPool (hence `volatile`) to decide whether to+ // retire the container instead of pooling it.+ //+ // Distinct from `stopped`: this does *not* tear the server down. The invocation already+ // in flight when the extension failed still completes normally through the runtime's own+ // /response call, matching AWS's treatment of the environment as condemned for future work+ // rather than aborted mid-invoke. WarmPool performs the actual teardown afterwards.+ private volatile boolean faulted;++ // Init-readiness barrier. ContainerLauncher launches extension binaries as detached `docker+ // exec`s and returns immediately, so without this the first invocation can reach the runtime+ // before an extension is ready — the adapter never sees that invoke and silently misses it.+ // ContainerLauncher declares how many binaries it launched via expectExtensions();+ // awaitExtensionsReady() blocks the launch until they are all ready (or the deadline passes).+ //+ // Readiness is the extension's *first /extension/event/next*, not its register call, matching+ // AWS's lifecycle: registering only obtains an identifier, whereas an extension that is+ // polling for events has finished its own initialisation and can actually receive an INVOKE.+ // An extension that registers and then spends time initialising is correctly still counted as+ // not-ready during that gap.+ //+ // A CountDownLatch is level-triggered, so an await() that arrives after the last extension+ // became ready returns immediately rather than missing the signal — the latch is created+ // before any extension can register, which is what makes that safe.+ private volatile CountDownLatch extensionsReady;++ // Set by ContainerLauncher once it knows which function this server instance is for (the+ // factory creates the server generically, before that's known) — used only to populate the+ // Extensions API register response.+ private volatile String functionName = "function";+ private volatile String functionVersion = "$LATEST";+ private volatile String handler = "";+ private volatile String accountId = DEFAULT_ACCOUNT_ID;RuntimeApiServer(Vertx vertx, int port) {this.vertx = vertx;this.port = port;}+ public void setFunctionMetadata(String functionName, String functionVersion, String handler,+ String accountId) {+ this.functionName = functionName != null ? functionName : "function";+ this.functionVersion = functionVersion != null ? functionVersion : "$LATEST";+ this.handler = handler != null ? handler : "";+ this.accountId = accountId != null && !accountId.isBlank() ? accountId : DEFAULT_ACCOUNT_ID;+ }+public int getPort() {return port;}+ /**+ * True once a registered extension reported an init or exit error. The execution environment+ * is condemned: {@code WarmPool} must not return this container to the pool or hand it out+ * again, matching real AWS's treatment of both errors as fatal to the environment.+ */+ public boolean isFaulted() {+ return faulted;+ }++ /**+ * Declares how many extension binaries were launched for this container, arming the+ * init-readiness barrier that {@link #awaitExtensionsReady(long)} waits on.+ *+ * <p>Must be called <em>before</em> the extension processes are started, so the latch exists+ * before any of them can call {@code /extension/register}. A count of zero (the common case:+ * no {@code /opt/extensions} directory) leaves the barrier permanently open.+ */+ public void expectExtensions(int count) {+ extensionsReady = new CountDownLatch(Math.max(0, count));+ }++ /**+ * Blocks until every extension declared via {@link #expectExtensions(int)} is init-ready —+ * that is, until each has issued its first {@code /extension/event/next} — or the timeout+ * elapses.+ *+ * @return true if all expected extensions became ready; false if the wait timed out with some+ * still missing, in which case the container is still usable — it simply starts+ * serving invocations without the extensions that never checked in.+ */+ public boolean awaitExtensionsReady(long timeoutMs) throws InterruptedException {+ CountDownLatch latch = extensionsReady;+ if (latch == null) {+ return true;+ }+ return latch.await(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS);+ }+public CompletableFuture<Void> start() {CompletableFuture<Void> started = new CompletableFuture<>();@@ -76,34 +229,26 @@ public class RuntimeApiServer {// Uses a reactive pattern (no thread held while waiting) to avoid Vert.x worker pool// exhaustion when many warm containers poll concurrently.router.get(NEXT_PATH).handler(ctx -> {- if (stopped) {- ctx.response().setStatusCode(204).end();- return;- }- PendingInvocation invocation = pendingQueue.poll();- if (invocation != null) {- if (stopped) {- invocation.getResultFuture().complete(- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));- ctx.response().setStatusCode(204).end();- return;+ PendingInvocation toDispatch = null;+ boolean send204 = false;+ synchronized (lock) {+ // `faulted` alongside `stopped`: a condemned environment must not be handed work,+ // including an invocation that was queued just before the fault was reported.+ if (stopped || faulted) {+ send204 = true;+ } else {+ toDispatch = pendingQueue.poll();+ if (toDispatch == null) {+ waitingContexts.add(ctx);+ }}- sendInvocation(ctx, invocation);- return;}- // No invocation queued yet — park this context until enqueue() wakes it.- waitingContexts.add(ctx);- // Re-check stop race: stop() may have drained waitingContexts before our add().- if (stopped && waitingContexts.remove(ctx)) {+ if (send204) {ctx.response().setStatusCode(204).end();- return;+ } else if (toDispatch != null) {+ sendInvocation(ctx, toDispatch);}- // Re-check enqueue race: an invocation may have arrived between our poll() and add().- PendingInvocation raced = pendingQueue.poll();- if (raced != null && waitingContexts.remove(ctx)) {- sendInvocation(ctx, raced);- }- // else: still parked — enqueue() will dispatch via vertx.runOnContext().+ // else: parked — enqueue() or stop() will dispatch this ctx later.});// POST /runtime/invocation/{requestId}/response — success@@ -138,6 +283,143 @@ public class RuntimeApiServer {sendStatusOk(ctx);});+ // POST /extension/register — an extension process (e.g. aws-lambda-web-adapter)+ // registers to receive lifecycle events. Real AWS requires the Lambda-Extension-Name+ // header to equal the extension's own file name; floci does not validate that here+ // (the identifier it hands back is sufficient for the extension to poll /event/next).+ router.post(EXTENSION_REGISTER_PATH).handler(ctx -> {+ String name = ctx.request().getHeader(EXTENSION_NAME_HEADER);+ if (name == null || name.isBlank()) {+ ctx.response().setStatusCode(400)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorMessage\":\"Missing Lambda-Extension-Name header\"}");+ return;+ }+ List<String> events = List.of("INVOKE", "SHUTDOWN");+ var body = ctx.body().asJsonObject();+ if (body != null && body.getJsonArray("events") != null) {+ events = body.getJsonArray("events").stream().map(String::valueOf).toList();+ }+ String identifier = UUID.randomUUID().toString();+ RegisteredExtension extension = new RegisteredExtension(identifier, name, events);+ boolean environmentFaulted;+ synchronized (lock) {+ // AWS makes an init/exit error terminal for the whole Extensions API, not just for+ // runtime work: once the environment is condemned no further extension call+ // succeeds. Registering into a container about to be retired would hand back an+ // identifier that can never receive an event.+ //+ // Checked *inside* the lock, like every other guard on this flag:+ // handleExtensionFatalError sets `faulted` under the lock, so a volatile read+ // before acquiring it can be overtaken between the check and the put() below,+ // registering into an environment condemned a moment later.+ environmentFaulted = faulted;+ if (!environmentFaulted) {+ extensions.put(identifier, extension);+ if (stopped && extension.isSubscribedTo(ExtensionEvent.Type.SHUTDOWN)) {+ // The server is already stopping — this extension will never see stop()'s+ // own SHUTDOWN fan-out (that already ran), so queue one now to preserve+ // "every registered extension eventually sees SHUTDOWN."+ extension.getPendingEvents().offer(+ ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN"));+ }+ }+ }+ if (environmentFaulted) {+ sendExtensionFaulted(ctx);+ return;+ }+ LOG.infov("Extension registered: {0} ({1}), events={2}", name, identifier, events);++ JsonObject responseBody = new JsonObject()+ .put("functionName", functionName)+ .put("functionVersion", functionVersion)+ .put("handler", handler);+ // AWS only includes accountId when the extension opts in via Lambda-Extension-Accept-+ // Feature; adding it unconditionally would diverge from the real response shape for+ // extensions that never asked. The header is a comma-separated feature list.+ if (acceptsFeature(ctx.request().getHeader(EXTENSION_ACCEPT_FEATURE_HEADER), ACCOUNT_ID_FEATURE)) {+ responseBody.put("accountId", accountId);+ }+ ctx.response()+ .setStatusCode(200)+ .putHeader(EXTENSION_ID_HEADER, identifier)+ .putHeader("Content-Type", "application/json")+ .end(responseBody.encode());+ });++ // GET /extension/event/next — blocks until the next INVOKE or SHUTDOWN event for this+ // extension. Mirrors NEXT_PATH's park/dispatch pattern, scoped per-extension since each+ // extension has its own independent event queue.+ router.get(EXTENSION_NEXT_PATH).handler(ctx -> {+ String identifier = ctx.request().getHeader(EXTENSION_ID_HEADER);+ ExtensionEvent toDispatch = null;+ boolean send204 = false;+ boolean unknownExtension = false;+ boolean environmentFaulted = false;+ CountDownLatch readyLatch = null;+ synchronized (lock) {+ RegisteredExtension extension = identifier != null ? extensions.get(identifier) : null;+ if (extension == null) {+ unknownExtension = true;+ } else if (faulted) {+ // Read under the lock alongside `stopped`: handleExtensionFatalError() sets+ // `faulted` and drains the parked pollers as one atomic change, so checking it+ // here is what stops a poller parking again straight after that drain.+ environmentFaulted = true;+ } else {+ // AWS treats an extension as init-ready at its first /event/next, not at+ // register: the extension has finished its own initialisation only once it+ // starts polling. Count the readiness barrier down here, once per extension.+ if (extension.markFirstNextReceived()) {+ readyLatch = extensionsReady;+ }+ // Poll before checking `stopped`: stop() offers a SHUTDOWN to pendingEvents for+ // any extension it doesn't find already parked, so a request arriving just+ // after `stopped` was set — but after that offer landed — must still see it.+ // Checking `stopped` first would answer 204 and orphan the queued SHUTDOWN.+ toDispatch = extension.getPendingEvents().poll();+ if (toDispatch == null) {+ // Deliberate deviation: AWS documents only 200/403/500 here and never ends+ // the long-poll with an empty response. Floci answers 204 once the server+ // is stopping and there is genuinely nothing left to deliver, because the+ // alternative is holding the connection open while the listener is torn+ // down — the extension would see a dropped socket instead of a clean+ // close. Only reachable for an extension not subscribed to SHUTDOWN (a+ // subscribed one gets the real SHUTDOWN event from stop()'s fan-out, which+ // the poll above returns), so no extension that asked to be told about+ // shutdown learns about it this way.+ if (stopped) {+ send204 = true;+ } else {+ extension.setWaitingContext(ctx);+ }+ }+ }+ }+ if (readyLatch != null) {+ readyLatch.countDown();+ }+ if (unknownExtension) {+ ctx.response().setStatusCode(403)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");+ } else if (environmentFaulted) {+ sendExtensionFaulted(ctx);+ } else if (send204) {+ ctx.response().setStatusCode(204).end();+ } else if (toDispatch != null) {+ sendExtensionEvent(ctx, toDispatch);+ }+ // else: parked — notifyExtensionsOfInvoke()/stop() will dispatch this ctx later.+ });++ // POST /extension/init/error, /extension/exit/error — an extension reports it can't+ // continue. Floci doesn't restart the execution environment on this (unlike real AWS);+ // it's enough to unregister the extension so future event fan-out skips it.+ router.post(EXTENSION_INIT_ERROR_PATH).handler(ctx -> handleExtensionFatalError(ctx, "init"));+ router.post(EXTENSION_EXIT_ERROR_PATH).handler(ctx -> handleExtensionFatalError(ctx, "exit"));+long deadline = System.currentTimeMillis() + 5000;tryListen(started, router, deadline);@@ -164,13 +446,44 @@ public class RuntimeApiServer {});}- public synchronized CompletableFuture<Void> stop() {- if (closeFuture != null) {- return closeFuture;+ public CompletableFuture<Void> stop() {+ CompletableFuture<Void> closed;+ List<RoutingContext> contextsToClose204;+ List<Runnable> dispatches = new ArrayList<>();+ List<PendingInvocation> invocationsToFailContainerStopped;++ synchronized (lock) {+ if (closeFuture != null) {+ return closeFuture;+ }+ stopped = true;+ closed = new CompletableFuture<>();+ closeFuture = closed;++ contextsToClose204 = new ArrayList<>(waitingContexts);+ waitingContexts.clear();++ ExtensionEvent shutdownEvent = ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN");+ for (RegisteredExtension ext : extensions.values()) {+ if (!ext.isSubscribedTo(ExtensionEvent.Type.SHUTDOWN)) {+ continue;+ }+ RoutingContext waitingCtx = ext.takeWaitingContext();+ if (waitingCtx != null) {+ dispatches.add(() -> {+ if (!waitingCtx.response().ended()) {+ sendExtensionEvent(waitingCtx, shutdownEvent);+ }+ });+ } else {+ ext.getPendingEvents().offer(shutdownEvent);+ }+ }++ invocationsToFailContainerStopped = new ArrayList<>(pendingQueue);+ pendingQueue.clear();}- stopped = true;- CompletableFuture<Void> closed = new CompletableFuture<>();- closeFuture = closed;+if (httpServer != null) {httpServer.close(ar -> {if (ar.succeeded()) {@@ -186,9 +499,7 @@ public class RuntimeApiServer {}// Wake any parked /next pollers with 204 (container shutting down — runtime will exit).- RoutingContext waiting;- while ((waiting = waitingContexts.poll()) != null) {- final RoutingContext ctx = waiting;+ for (RoutingContext ctx : contextsToClose204) {vertx.runOnContext(v -> {if (!ctx.response().ended()) {ctx.response().setStatusCode(204).end();@@ -196,9 +507,13 @@ public class RuntimeApiServer {});}- // Drain queued invocations that were never consumed by /next.- PendingInvocation pending;- while ((pending = pendingQueue.poll()) != null) {+ // Dispatch SHUTDOWN to every extension that was already parked on /event/next.+ for (Runnable dispatch : dispatches) {+ vertx.runOnContext(v -> dispatch.run());+ }++ // Fail queued invocations that were never consumed by /next.+ for (PendingInvocation pending : invocationsToFailContainerStopped) {pending.getResultFuture().complete(new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));}@@ -213,35 +528,93 @@ public class RuntimeApiServer {}public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {- if (stopped) {+ boolean rejected;+ List<Runnable> dispatches = new ArrayList<>();+ RoutingContext waitingCtxForInvocation = null;++ synchronized (lock) {+ // A condemned environment (an extension reported init/exit error) refuses new work the+ // same way a stopping one does. Without this the invocation is queued to a runtime that+ // will never be given it, and the caller hangs until the function timeout rather than+ // failing fast. Read inside the lock alongside `stopped` so the accept/reject decision+ // stays a single atomic step.+ rejected = stopped || faulted;+ if (!rejected) {+ if (!extensions.isEmpty()) {+ ExtensionEvent event = ExtensionEvent.invoke(+ invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());+ for (RegisteredExtension ext : extensions.values()) {+ if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {+ continue;+ }+ RoutingContext waitingCtx = ext.takeWaitingContext();+ if (waitingCtx != null) {+ dispatches.add(() -> {+ if (!waitingCtx.response().ended()) {+ sendExtensionEvent(waitingCtx, event);+ } else {+ synchronized (lock) {+ ext.getPendingEvents().offer(event);+ }+ }+ });+ } else {+ ext.getPendingEvents().offer(event);+ }+ }+ }++ waitingCtxForInvocation = waitingContexts.poll();+ if (waitingCtxForInvocation == null) {+ pendingQueue.offer(invocation);+ }+ }+ }++ if (rejected) {invocation.getResultFuture().complete(new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));return invocation.getResultFuture();}- // If a /next poller is already parked, dispatch immediately on the event loop.- RoutingContext ctx = waitingContexts.poll();- if (ctx != null) {- final RoutingContext waitingCtx = ctx;++ for (Runnable dispatch : dispatches) {+ vertx.runOnContext(v -> dispatch.run());+ }++ if (waitingCtxForInvocation != null) {+ final RoutingContext waitingCtx = waitingCtxForInvocation;vertx.runOnContext(v -> {if (!waitingCtx.response().ended()) {sendInvocation(waitingCtx, invocation);} else {// Connection closed between park and dispatch — re-queue.- pendingQueue.offer(invocation);+ synchronized (lock) {+ pendingQueue.offer(invocation);+ }}});- return invocation.getResultFuture();- }- pendingQueue.offer(invocation);- // Close the check-then-offer race: if stop() ran between the guard and offer(),- // the drain is done and our invocation would sit forever. Remove and complete.- if (stopped && pendingQueue.remove(invocation)) {- invocation.getResultFuture().complete(- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));}return invocation.getResultFuture();}+ /**+ * True if {@code headerValue} — the comma-separated {@code Lambda-Extension-Accept-Feature}+ * list an extension sends at registration — opts into {@code feature}. Matching is+ * case-insensitive and tolerant of surrounding whitespace; a null/blank header opts into+ * nothing.+ */+ private static boolean acceptsFeature(String headerValue, String feature) {+ if (headerValue == null || headerValue.isBlank()) {+ return false;+ }+ for (String candidate : headerValue.split(",")) {+ if (candidate.trim().equalsIgnoreCase(feature)) {+ return true;+ }+ }+ return false;+ }+private void sendStatusOk(RoutingContext ctx) {ctx.response().setStatusCode(202)@@ -251,7 +624,7 @@ public class RuntimeApiServer {private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {inFlight.put(invocation.getRequestId(), invocation);-+byte[] payload = invocation.getPayload();String body = (payload != null && payload.length > 0)? new String(payload)@@ -264,4 +637,140 @@ public class RuntimeApiServer {.putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs())).end(body);}++ private void sendExtensionEvent(RoutingContext ctx, ExtensionEvent event) {+ JsonObject body = new JsonObject().put("eventType", event.getType().name());+ if (event.getType() == ExtensionEvent.Type.INVOKE) {+ body.put("requestId", event.getRequestId())+ .put("invokedFunctionArn", event.getFunctionArn())+ .put("deadlineMs", event.getDeadlineMs());+ } else {+ body.put("shutdownReason", event.getShutdownReason())+ .put("deadlineMs", event.getDeadlineMs());+ }+ ctx.response()+ .setStatusCode(200)+ .putHeader("Content-Type", "application/json")+ .putHeader(EXTENSION_EVENT_ID_HEADER, UUID.randomUUID().toString())+ .end(body.encode());+ }++ /**+ * Response for any Extensions API call made after an extension reported an init/exit error.+ * AWS makes the fault terminal for the whole environment — no further extension call succeeds+ * — so this is a hard error rather than the 204 used for an orderly shutdown, which an+ * extension is entitled to read as "nothing right now, poll again".+ */+ private void sendExtensionFaulted(RoutingContext ctx) {+ ctx.response()+ .setStatusCode(500)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorType\":\"Extension.SandboxFaulted\","+ + "\"errorMessage\":\"Execution environment condemned by an extension "+ + "init/exit error\"}");+ }++ private void handleExtensionFatalError(RoutingContext ctx, String phase) {+ String identifier = ctx.request().getHeader(EXTENSION_ID_HEADER);+ String errorType = ctx.request().getHeader(EXTENSION_ERROR_TYPE_HEADER);++ // Validate before mutating anything. Retiring the execution environment is destructive and+ // unrecoverable, so a malformed report must not trigger it — otherwise a caller that got+ // the contract wrong silently kills a healthy container and the 202 tells them it worked.+ // AWS requires Lambda-Extension-Function-Error-Type on both error endpoints, and rejects+ // an unknown identifier rather than treating it as a fatal report.+ if (identifier == null || identifier.isBlank()) {+ ctx.response().setStatusCode(403)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");+ return;+ }+ if (errorType == null || errorType.isBlank()) {+ ctx.response().setStatusCode(400)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorMessage\":\"Missing Lambda-Extension-Function-Error-Type header\"}");+ return;+ }++ RegisteredExtension removed;+ List<PendingInvocation> strandedInvocations;+ List<RoutingContext> strandedPollers;+ List<RoutingContext> strandedExtensionPollers;+ synchronized (lock) {+ removed = extensions.remove(identifier);+ if (removed == null) {+ // Unknown identifier: reject without condemning. Answered outside the lock below.+ strandedInvocations = List.of();+ strandedPollers = List.of();+ strandedExtensionPollers = List.of();+ } else {+ // Set under the lock alongside the unregistration so both land as one atomic+ // change.+ faulted = true;++ // Anything already queued or parked would otherwise wait forever, since the guards+ // in enqueue()/NEXT_PATH now refuse to move work through a condemned environment.+ // Drain both here so callers fail fast instead of hanging until the function+ // timeout.+ strandedInvocations = new ArrayList<>(pendingQueue);+ pendingQueue.clear();+ strandedPollers = new ArrayList<>(waitingContexts);+ waitingContexts.clear();++ // Sibling extensions parked on /event/next need the same treatment, and for the+ // same reason: the faulted guard in that handler means nothing will ever wake+ // them, so without this their long-poll stays open until stop() eventually runs+ // — up to a full function timeout away. Mirrors the extensions.values() sweep in+ // stop(); harmless with a single extension, but a second one hangs without it.+ strandedExtensionPollers = new ArrayList<>();+ for (RegisteredExtension ext : extensions.values()) {+ RoutingContext waitingCtx = ext.takeWaitingContext();+ if (waitingCtx != null) {+ strandedExtensionPollers.add(waitingCtx);+ }+ }+ }+ }++ if (removed == null) {+ LOG.warnv("Unknown extension identifier {0} reported {1} error on port {2}; ignoring",+ identifier, phase, String.valueOf(port));+ ctx.response().setStatusCode(403)+ .putHeader("Content-Type", "application/json")+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");+ return;+ }++ LOG.warnv("Extension {0} ({1}) reported {2} error ({3}) on port {4}; "+ + "retiring execution environment",+ removed.getName(), identifier, phase, errorType, String.valueOf(port));++ for (PendingInvocation stranded : strandedInvocations) {+ stranded.getResultFuture().complete(+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, stranded.getRequestId()));+ }+ for (RoutingContext poller : strandedPollers) {+ vertx.runOnContext(v -> {+ if (!poller.response().ended()) {+ poller.response().setStatusCode(204).end();+ }+ });+ }+ // Reuses the same 500 that /event/next returns to any extension polling after a fault,+ // rather than inventing a separate response for the already-parked case: AWS specifies+ // that the environment is terminal but not what an open long-poll receives, so matching+ // the behaviour we already ship is the conservative choice.+ for (RoutingContext poller : strandedExtensionPollers) {+ vertx.runOnContext(v -> {+ if (!poller.response().ended()) {+ sendExtensionFaulted(poller);+ }+ });+ }++ ctx.response()+ .setStatusCode(202)+ .putHeader("Content-Type", "application/json")+ .end(STATUS_OK_BODY);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.javaindex e1d10754f0..1315055ece 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.launcher.ContainerLauncher;import io.github.hectorvent.floci.services.lambda.model.ContainerState;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;import org.mockito.Mock;@@ -14,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.lenient;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;@@ -157,6 +159,80 @@ class WarmPoolTest {pool.shutdown();}+ /**+ * An extension reporting init/exit error is fatal to the execution environment in real AWS,+ * so the container must be retired after the invocation rather than returned to the pool.+ */+ @Test+ void releaseAfterExtensionFatalError_retiresContainerInsteadOfPooling() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("faulted-fn");++ RuntimeApiServer faultedServer = mock(RuntimeApiServer.class);+ when(faultedServer.isFaulted()).thenReturn(true);+ ContainerHandle faulted = new ContainerHandle("cid-faulted", "faulted-fn", faultedServer, ContainerState.WARM);+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "faulted-fn", null, ContainerState.WARM);++ // Both acquires below cold-start (the pool is empty, then the faulted handle is discarded+ // before any liveness check), so no isAlive stubbing is needed.+ when(containerLauncher.launch(any())).thenReturn(faulted, fresh);++ ContainerHandle first = pool.acquire(fn);+ assertSame(faulted, first);++ // The extension died during this invocation; releasing must tear the container down.+ pool.release(first);+ verify(containerLauncher, times(1)).stop(faulted);++ // The next acquire cold-starts rather than handing the condemned container back out.+ ContainerHandle second = pool.acquire(fn);+ assertSame(fresh, second);+ assertNotSame(faulted, second);+ verify(containerLauncher, times(2)).launch(any());++ pool.shutdown();+ }++ /**+ * A container whose extension faults while it sits WARM in the pool is still *running*, so the+ * liveness probe alone would hand it back out. It must be skipped and discarded on acquire.+ */+ @Test+ void acquire_discardsPooledHandleWhoseExtensionFaulted() {+ WarmPool pool = buildPool();+ pool.init();++ LambdaFunction fn = mock(LambdaFunction.class);+ when(fn.getFunctionName()).thenReturn("pooled-fault-fn");++ RuntimeApiServer server = mock(RuntimeApiServer.class);+ ContainerHandle pooled = new ContainerHandle("cid-pooled", "pooled-fault-fn", server, ContainerState.WARM);+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "pooled-fault-fn", null, ContainerState.WARM);++ when(containerLauncher.launch(any())).thenReturn(pooled, fresh);+ // Healthy at release time, so it goes into the pool as normal.+ when(server.isFaulted()).thenReturn(false);+ ContainerHandle seeded = pool.acquire(fn);+ assertSame(pooled, seeded);+ pool.release(seeded);++ // The extension now faults while the container sits idle in the pool. isAlive() is stubbed+ // true so the container is unambiguously still running: if the faulted check were removed,+ // this handle would be considered reusable and handed straight back out.+ when(server.isFaulted()).thenReturn(true);+ lenient().when(containerLauncher.isAlive(pooled)).thenReturn(true);++ ContainerHandle acquired = pool.acquire(fn);+ assertSame(fresh, acquired);+ assertNotSame(pooled, acquired);+ verify(containerLauncher, times(1)).stop(pooled);++ pool.shutdown();+ }+@Testvoid acquire_discardsDeadPooledHandleAndColdStarts() {WarmPool pool = buildPool();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.javaindex ac14850c91..e767f6f109 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java@@ -14,9 +14,19 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.command.ExecCreateCmd;+import com.github.dockerjava.api.command.ExecCreateCmdResponse;+import com.github.dockerjava.api.command.ExecStartCmd;+import com.github.dockerjava.api.command.CopyArchiveFromContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.Frame;+import com.github.dockerjava.api.model.StreamType;import com.github.dockerjava.api.model.Mount;import com.github.dockerjava.api.model.MountType;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;@@ -26,11 +36,16 @@ import org.mockito.InOrder;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;+import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;++import java.io.IOException;+import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;@@ -38,6 +53,8 @@ import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyLong;+import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.*;@@ -565,4 +582,334 @@ class ContainerLauncherTest {verify(lifecycleManager, never()).create(any());verify(lifecycleManager, never()).startCreated(any(), any());}++ /** Builds a tar archive matching what {@code docker cp}/{@code copyArchiveFromContainerCmd}+ * returns for a directory: the directory itself as a leading entry, then each name as a direct+ * child, executable. */+ private static byte[] tarOf(String... binaryNames) throws IOException {+ var entries = new java.util.LinkedHashMap<String, Boolean>();+ for (String name : binaryNames) {+ entries.put(name, true);+ }+ return tarOfRaw(entries);+ }++ /** Like {@link #tarOf}, but lets each entry's executable bit be controlled individually+ * (name -> executable), so filtering logic (non-executable files, nested paths) can be+ * exercised. Names containing "/" are written as-is (not prefixed), to model entries nested+ * more than one level below the extensions directory. */+ private static byte[] tarOfRaw(java.util.Map<String, Boolean> nameToExecutable) throws IOException {+ var bos = new java.io.ByteArrayOutputStream();+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(bos)) {+ TarArchiveEntry dir = new TarArchiveEntry("extensions/");+ tar.putArchiveEntry(dir);+ tar.closeArchiveEntry();+ for (var e : nameToExecutable.entrySet()) {+ TarArchiveEntry entry = new TarArchiveEntry("extensions/" + e.getKey());+ entry.setMode(e.getValue() ? 0100755 : 0100644); // executable vs. non-executable regular file+ entry.setSize(0);+ tar.putArchiveEntry(entry);+ tar.closeArchiveEntry();+ }+ }+ return bos.toByteArray();+ }++ /** Stubs container-123's /opt/extensions listing (via the Docker archive API, not exec) to+ * return the given binary names, and stubs exec of each resulting /opt/extensions/<name> path+ * to succeed immediately. */+ private List<ExecCreateCmd> stubExtensionDiscovery(String... binaryNames) throws IOException {+ return stubExtensionDiscoveryFromTar(tarOf(binaryNames));+ }++ /** Same as {@link #stubExtensionDiscovery}, but takes a caller-built tar so tests can exercise+ * {@code listExtensionBinaries}' entry filtering (non-executable files, nested paths) rather+ * than only the convenience all-executable-direct-children shape. Returns the {@code ExecCreateCmd}+ * mock for each launched binary, in launch order — pass it to {@link #capturedLaunchPaths} after+ * calling {@code launch()} to assert on exactly which discovered names were (and weren't) launched. */+ private List<ExecCreateCmd> stubExtensionDiscoveryFromTar(byte[] tar) {+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);+ when(copyCmd.exec()).thenReturn(new java.io.ByteArrayInputStream(tar));+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))+ .thenReturn(copyCmd);++ List<ExecCreateCmd> launchCmds = new java.util.ArrayList<>();+ lenient().when(dockerClient.execCreateCmd("container-123"))+ .thenAnswer(invocation -> {+ ExecCreateCmd launchCmd = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));+ ExecCreateCmdResponse launchResponse = mock(ExecCreateCmdResponse.class);+ when(launchResponse.getId()).thenReturn("exec-launch-" + java.util.UUID.randomUUID());+ when(launchCmd.exec()).thenReturn(launchResponse);+ launchCmds.add(launchCmd);+ return launchCmd;+ });+ lenient().when(dockerClient.execStartCmd(argThat(id -> id != null && id.startsWith("exec-launch-"))))+ .thenAnswer(invocation -> {+ ExecStartCmd start = mock(ExecStartCmd.class);+ when(start.exec(any())).thenAnswer(startInvocation -> {+ @SuppressWarnings("unchecked")+ ResultCallback<Frame> cb = startInvocation.getArgument(0);+ cb.onComplete();+ return cb;+ });+ return start;+ });+ // launcher.launch(fn) populates this list after this method returns.+ return launchCmds;+ }++ /** Overrides the default exec-start stub so the callback the launcher supplies is driven with a+ * real STDOUT {@link Frame} carrying {@code output}, exercising the frame-to-CloudWatch path+ * rather than only completing the callback. */+ private void stubExecStartEmittingFrame(String output) {+ lenient().when(dockerClient.execStartCmd(argThat(id -> id != null && id.startsWith("exec-launch-"))))+ .thenAnswer(invocation -> {+ ExecStartCmd start = mock(ExecStartCmd.class);+ when(start.exec(any())).thenAnswer(startInvocation -> {+ @SuppressWarnings("unchecked")+ ResultCallback<Frame> cb = startInvocation.getArgument(0);+ cb.onNext(new Frame(StreamType.STDOUT, output.getBytes(StandardCharsets.UTF_8)));+ cb.onComplete();+ return cb;+ });+ return start;+ });+ }++ /** Extracts the single command-line argument each mock in {@code launchCmds} was called with+ * via {@code withCmd(String...)}, in call order — the {@code /opt/extensions/<name>} path+ * each discovered extension binary was launched with. */+ private static List<String> capturedLaunchPaths(List<ExecCreateCmd> launchCmds) {+ List<String> paths = new java.util.ArrayList<>();+ for (ExecCreateCmd cmd : launchCmds) {+ ArgumentCaptor<String[]> captor = ArgumentCaptor.forClass(String[].class);+ verify(cmd).withCmd(captor.capture());+ paths.add(captor.getValue()[0]);+ }+ return paths;+ }++ @Test+ void launchFunction_discoversAndLaunchesExtensionBinaries() throws Exception {+ stubExtensionDiscovery("lambda-adapter");++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("with-extension-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ // The extension binary was discovered via the archive API and execed by its full+ // /opt/extensions path, after the container was started (real AWS starts extensions once+ // the container is up, alongside the runtime).+ verify(dockerClient).copyArchiveFromContainerCmd("container-123", "/opt/extensions");+ verify(dockerClient, atLeastOnce()).execCreateCmd(eq("container-123"));++ InOrder inOrder = inOrder(lifecycleManager, dockerClient);+ inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());+ inOrder.verify(dockerClient).copyArchiveFromContainerCmd("container-123", "/opt/extensions");+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");+ }++ /**+ * Extension stdout/stderr must reach the function's CloudWatch log group. `docker logs` — and+ * therefore ContainerLogStreamer.attach(), which uses logContainerCmd — only covers the+ * container's PID 1 output, so an exec's stream never reaches the container log. Without+ * explicit forwarding an observability extension's output is dropped entirely.+ *+ * <p>Uses a real ContainerLogStreamer over a mocked CloudWatchLogsService so the assertion+ * covers the actual frame-to-log-event path rather than just that a mock was called.+ */+ @Test+ void launchFunction_forwardsExtensionOutputToCloudWatchLogs() throws Exception {+ CloudWatchLogsService cloudWatchLogs = mock(CloudWatchLogsService.class);+ ContainerReachableEndpoint reachableEndpoint =+ new ContainerReachableEndpoint(config, dockerHostResolver, embeddedDnsServer);+ ContainerLauncher launcherWithRealStreamer = new ContainerLauncher(+ new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer),+ lifecycleManager,+ new ContainerLogStreamer(dockerClient, cloudWatchLogs),+ imageResolver, runtimeApiServerFactory, dockerHostResolver, config,+ ecrRegistryManager,+ mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class),+ new LaunchedContainerAwsEnv(reachableEndpoint));++ stubExtensionDiscovery("otel-collector");+ // Feed a real stdout frame through whatever callback the launcher hands to execStartCmd.+ stubExecStartEmittingFrame("extension started on :8080\n");++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("observability-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcherWithRealStreamer.launch(fn);++ // The frame became a CloudWatch log event in the function's own log group.+ ArgumentCaptor<List<Map<String, Object>>> events = ArgumentCaptor.forClass(List.class);+ verify(cloudWatchLogs, atLeastOnce()).putLogEvents(+ eq("/aws/lambda/observability-fn"), anyString(), events.capture(), anyString());+ assertTrue(events.getAllValues().stream()+ .flatMap(List::stream)+ .anyMatch(e -> "extension started on :8080".equals(e.get("message"))),+ "extension stdout must be forwarded to the function's CloudWatch log group");+ }++ /**+ * The log group and stream must exist before extensions are launched: they can log immediately,+ * and putLogEvents against a missing stream is swallowed at debug level — silently losing+ * exactly the early startup output this forwarding exists to capture.+ */+ @Test+ void launchFunction_createsLogGroupBeforeLaunchingExtensions() throws Exception {+ stubExtensionDiscovery("lambda-adapter");++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("ordering-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ InOrder inOrder = inOrder(logStreamer, dockerClient);+ inOrder.verify(logStreamer).ensureLogGroupAndStream(+ eq("/aws/lambda/ordering-fn"), anyString(), anyString());+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");+ }++ /**+ * The init-readiness barrier abanna asked for on PR #1773: extension processes are started as+ * detached execs, so without waiting the caller could enqueue the first invocation before an+ * extension was ready for it and the adapter would silently miss that invoke. The launch must+ * arm the barrier with the discovered binary count *before* starting any exec (so a+ * fast-starting extension can't become ready before there is a latch to count it down), and+ * must not return until the extensions are init-ready.+ */+ @Test+ void launchFunction_waitsForExtensionsToBecomeReadyBeforeReturning() throws Exception {+ stubExtensionDiscovery("lambda-adapter", "otel-collector");++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("barrier-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ // Readiness is delayed: awaitExtensionsReady only reports success after a pause, standing+ // in for extension processes that take a moment to start up and poll for their first event.+ AtomicBoolean readinessComplete = new AtomicBoolean(false);+ when(runtimeApiServer.awaitExtensionsReady(anyLong())).thenAnswer(inv -> {+ Thread.sleep(150);+ readinessComplete.set(true);+ return true;+ });++ launcher.launch(fn);++ assertTrue(readinessComplete.get(),+ "launch() must not return before the extensions are init-ready");++ // The barrier is armed with the number of binaries actually discovered, and armed before+ // any extension process is started.+ verify(runtimeApiServer).expectExtensions(2);+ InOrder inOrder = inOrder(runtimeApiServer, dockerClient);+ inOrder.verify(runtimeApiServer).expectExtensions(2);+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");+ inOrder.verify(runtimeApiServer).awaitExtensionsReady(anyLong());+ }++ /**+ * A slow or crashed extension must degrade to "invocations run without it", not fail the whole+ * function launch — the same outcome as before the barrier existed.+ */+ @Test+ void launchFunction_extensionReadinessTimeout_doesNotFailLaunch() throws Exception {+ stubExtensionDiscovery("never-registers");++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("timeout-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ when(runtimeApiServer.awaitExtensionsReady(anyLong())).thenReturn(false);++ ContainerHandle handle = launcher.launch(fn);++ assertNotNull(handle, "a readiness timeout must not fail the launch");+ assertEquals("container-123", handle.getContainerId());+ }++ /**+ * The common case — no /opt/extensions directory — must not wait at all: the barrier is armed+ * with zero and the launch proceeds immediately.+ */+ @Test+ void launchFunction_noExtensions_armsBarrierWithZeroAndDoesNotBlock() throws Exception {+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);+ when(copyCmd.exec()).thenThrow(new NotFoundException("no such directory"));+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))+ .thenReturn(copyCmd);++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("no-extension-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ verify(runtimeApiServer).expectExtensions(0);+ }++ @Test+ void launchFunction_noExtensionsDirectory_doesNotFailLaunch() throws Exception {+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);+ when(copyCmd.exec()).thenThrow(new NotFoundException("no such directory"));+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))+ .thenReturn(copyCmd);++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("no-extension-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ // The discovery probe still runs (best-effort), but nothing beyond it — no extension+ // binary path is ever execed since the directory doesn't exist.+ verify(dockerClient, never()).execCreateCmd("container-123");+ }++ @Test+ void launchFunction_extensionDiscovery_filtersNonExecutableAndNestedEntries() throws Exception {+ var entries = new java.util.LinkedHashMap<String, Boolean>();+ entries.put("lambda-adapter", true); // direct child, executable: launched+ entries.put("README.md", false); // direct child, not executable: skipped+ entries.put("nested/inner-binary", true); // nested (not a direct child): skipped+ List<ExecCreateCmd> launchCmds = stubExtensionDiscoveryFromTar(tarOfRaw(entries));++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("mixed-extensions-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ assertEquals(List.of("/opt/extensions/lambda-adapter"), capturedLaunchPaths(launchCmds),+ "only the direct, executable entry should be launched");+ }++ @Test+ void launchFunction_multipleExtensionBinaries_allDiscoveredAndLaunched() throws Exception {+ List<ExecCreateCmd> launchCmds = stubExtensionDiscoveryFromTar(tarOf("lambda-adapter", "otel-collector"));++ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("multi-extension-fn");+ fn.setPackageType("Image");+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");++ launcher.launch(fn);++ assertEquals(List.of("/opt/extensions/lambda-adapter", "/opt/extensions/otel-collector"),+ capturedLaunchPaths(launchCmds));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.javaindex fed89321ac..eaaf27d637 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java@@ -17,6 +17,8 @@ import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;+import java.util.ArrayList;+import java.util.List;import java.util.concurrent.CompletableFuture;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;@@ -28,6 +30,7 @@ import org.junit.jupiter.api.condition.OS;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -55,7 +58,13 @@ class RuntimeApiServerTest {void tearDown() throws Exception {server.stop().get(5, TimeUnit.SECONDS);scheduler.shutdownNow();- vertx.close();+ httpClient.close();+ // Await the close rather than firing it and moving on: Vertx.close() is asynchronous, so+ // an unawaited call lets the next test's setUp() create a new Vertx and bind a port while+ // this one's event loops and server sockets are still tearing down. Across a class with+ // stress tests that stand up hundreds of servers, that backlog surfaces as BindException+ // or a start() timeout in an unrelated test's setUp().+ vertx.close().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);}@Test@@ -343,9 +352,1024 @@ class RuntimeApiServerTest {assertEquals(404, resp.statusCode());}+ @Test+ @Timeout(10)+ void extensionRegister_returnsIdentifierHeaderAndFunctionBody() throws Exception {+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "lambda-adapter")+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[\"INVOKE\",\"SHUTDOWN\"]}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ assertTrue(response.headers().firstValue("Lambda-Extension-Identifier").isPresent(),+ "register must return a Lambda-Extension-Identifier header");+ JsonObject body = new JsonObject(response.body());+ assertNotNull(body.getString("functionName"));+ assertNotNull(body.getString("functionVersion"));+ }++ /**+ * Regression: the register response previously hardcoded functionName/functionVersion/handler+ * to placeholder values regardless of which function the server was actually serving — extensions+ * that key telemetry off this response (e.g. per-function metrics tagging) would mislabel every+ * function identically. ContainerLauncher calls setFunctionMetadata once it knows which+ * LambdaFunction a given RuntimeApiServer instance belongs to.+ */+ @Test+ @Timeout(10)+ void extensionRegister_returnsRealFunctionMetadataOnceSet() throws Exception {+ server.setFunctionMetadata("my-real-function", "3", "index.handler", "123456789012");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "lambda-adapter")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ JsonObject body = new JsonObject(response.body());+ assertEquals("my-real-function", body.getString("functionName"));+ assertEquals("3", body.getString("functionVersion"));+ assertEquals("index.handler", body.getString("handler"));+ }++ /**+ * AWS includes {@code accountId} in the register response when the extension opts in via+ * {@code Lambda-Extension-Accept-Feature: accountId}. Without it a telemetry extension+ * registers successfully but silently loses account attribution.+ */+ @Test+ @Timeout(10)+ void extensionRegister_withAccountIdFeature_includesAccountId() throws Exception {+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "telemetry-extension")+ .header("Lambda-Extension-Accept-Feature", "accountId")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ assertEquals("210987654321", new JsonObject(response.body()).getString("accountId"));+ }++ /**+ * The feature is opt-in: an extension that does not request it must get the unchanged response+ * shape, since real AWS omits the field entirely rather than always sending it.+ */+ @Test+ @Timeout(10)+ void extensionRegister_withoutAccountIdFeature_omitsAccountId() throws Exception {+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "plain-extension")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ assertNull(new JsonObject(response.body()).getString("accountId"),+ "accountId must only appear when the extension opts in");+ }++ /**+ * The header carries a comma-separated feature list, so accountId must still be honoured+ * alongside other requested features and regardless of casing/spacing.+ */+ @Test+ @Timeout(10)+ void extensionRegister_accountIdFeatureAmongOthers_isHonoured() throws Exception {+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "telemetry-extension")+ .header("Lambda-Extension-Accept-Feature", "someOtherFeature, AccountID")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ assertEquals("210987654321", new JsonObject(response.body()).getString("accountId"));+ }++ /** A function with no account falls back to the same placeholder used elsewhere in Floci. */+ @Test+ @Timeout(10)+ void extensionRegister_withNoFunctionAccountId_fallsBackToPlaceholder() throws Exception {+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", null);++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "telemetry-extension")+ .header("Lambda-Extension-Accept-Feature", "accountId")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode());+ assertEquals("000000000000", new JsonObject(response.body()).getString("accountId"));+ }++ @Test+ @Timeout(10)+ void extensionRegister_missingNameHeader_returns400() throws Exception {+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(400, response.statusCode());+ }++ @Test+ @Timeout(10)+ void extensionEventNext_unknownIdentifier_returns403() throws Exception {+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", "not-a-real-id")+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(403, response.statusCode());+ }++ @Test+ @Timeout(15)+ void extensionEventNext_receivesInvokeEventWhenRuntimeInvocationEnqueued() throws Exception {+ String extensionId = registerExtension("lambda-adapter", "INVOKE", "SHUTDOWN");++ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ Thread.sleep(300);+ assertFalse(asyncNext.isDone(), "extension /event/next should be parked with no pending event");++ PendingInvocation invocation = new PendingInvocation(+ "req-ext-invoke", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);+ assertEquals(200, response.statusCode());+ assertTrue(response.headers().firstValue("Lambda-Extension-Event-Identifier").isPresent());+ JsonObject body = new JsonObject(response.body());+ assertEquals("INVOKE", body.getString("eventType"));+ assertEquals("req-ext-invoke", body.getString("requestId"));+ }++ @Test+ @Timeout(15)+ void extensionEventNext_notSubscribedToInvoke_isNotNotified() throws Exception {+ // Registers for SHUTDOWN only — real AWS never delivers INVOKE to an extension that+ // didn't ask for it.+ String extensionId = registerExtension("shutdown-only-extension", "SHUTDOWN");++ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ server.enqueue(new PendingInvocation(+ "req-not-subscribed", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>()));++ Thread.sleep(500);+ assertFalse(asyncNext.isDone(),+ "extension not subscribed to INVOKE must not be woken by an invocation");+ }++ @Test+ @Timeout(15)+ void extensionEventNext_receivesShutdownEventWhenServerStops() throws Exception {+ String extensionId = registerExtension("lambda-adapter", "INVOKE", "SHUTDOWN");++ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ Thread.sleep(300);+ assertFalse(asyncNext.isDone());++ server.stop();++ HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);+ assertEquals(200, response.statusCode());+ JsonObject body = new JsonObject(response.body());+ assertEquals("SHUTDOWN", body.getString("eventType"));+ }++ @Test+ @Timeout(10)+ void extensionInitError_returns202AndUnregistersExtension() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.ConfigInvalid")+ .POST(HttpRequest.BodyPublishers.ofString(+ "{\"errorMessage\":\"bad config\",\"errorType\":\"Extension.ConfigInvalid\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(202, response.statusCode());+ assertEquals("OK", new JsonObject(response.body()).getString("status"));++ // The unregistered extension is no longer a valid target for /event/next.+ HttpResponse<String> nextResponse = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(403, nextResponse.statusCode());++ // Both init/error and exit/error are fatal to the execution environment in real AWS, so+ // the server is marked faulted for WarmPool to retire the container rather than reuse it.+ assertTrue(server.isFaulted(), "init/error must condemn the execution environment");+ }++ @Test+ @Timeout(30)+ void extensionExitError_condemnsExecutionEnvironment() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");+ assertFalse(server.isFaulted(), "environment must not be faulted before any error is reported");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString(+ "{\"errorMessage\":\"crashed\",\"errorType\":\"Extension.Crash\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(202, response.statusCode());+ assertTrue(server.isFaulted(), "exit/error must condemn the execution environment");+ }++ /**+ * The init-readiness barrier: awaitExtensionsReady() must block until every expected extension+ * is init-ready, which AWS defines as its first /extension/event/next.+ */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_blocksUntilAllExpectedExtensionsPollForEvents() throws Exception {+ server.expectExtensions(2);++ CompletableFuture<Boolean> awaited = CompletableFuture.supplyAsync(() -> {+ try {+ return server.awaitExtensionsReady(10_000);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return false;+ }+ });++ String first = registerExtension("first-extension", "INVOKE");+ String second = registerExtension("second-extension", "INVOKE");+ Thread.sleep(200);+ assertFalse(awaited.isDone(), "registering alone must not open the barrier");++ pollExtensionEventNext(first);+ Thread.sleep(200);+ assertFalse(awaited.isDone(), "must still be waiting while one extension has not polled");++ pollExtensionEventNext(second);+ assertTrue(awaited.get(5, TimeUnit.SECONDS), "must unblock once all extensions are ready");+ }++ /**+ * The distinction abanna raised on PR #1773: an extension that registers immediately but then+ * spends time initialising before its first Next is *not* ready, and the barrier must hold the+ * container out of service for that whole gap. Gating on register would release it early and+ * let the first invoke race the extension's own startup — the bug the barrier exists to stop.+ */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_waitsForDelayedFirstNextAfterFastRegistration() throws Exception {+ server.expectExtensions(1);+ String extensionId = registerExtension("slow-starting-extension", "INVOKE");++ CompletableFuture<Boolean> awaited = CompletableFuture.supplyAsync(() -> {+ try {+ return server.awaitExtensionsReady(10_000);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return false;+ }+ });++ Thread.sleep(700);+ assertFalse(awaited.isDone(),+ "a registered-but-not-yet-polling extension must keep the barrier closed");++ pollExtensionEventNext(extensionId);+ assertTrue(awaited.get(5, TimeUnit.SECONDS),+ "the delayed first /event/next must open the barrier");+ }++ /**+ * The missed-signal case: readiness completing *before* anyone waits must not strand the+ * launch. A CountDownLatch is level-triggered, so a late await returns immediately — this+ * pins that behaviour, since an edge-triggered signal here would hang the cold start.+ */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_returnsImmediatelyWhenExtensionAlreadyReady() throws Exception {+ server.expectExtensions(1);+ pollExtensionEventNext(registerExtension("fast-extension", "INVOKE"));++ long start = System.currentTimeMillis();+ assertTrue(server.awaitExtensionsReady(10_000));+ assertTrue(System.currentTimeMillis() - start < 1000,+ "an await arriving after readiness must not wait");+ }++ /**+ * Repeated polling must not count the barrier down more than once per extension, which would+ * let one chatty extension open a barrier that others have not reached yet.+ */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_countsEachExtensionOnceAcrossRepeatedPolls() throws Exception {+ server.expectExtensions(2);+ String chatty = registerExtension("chatty-extension", "INVOKE");+ registerExtension("quiet-extension", "INVOKE");++ for (int i = 0; i < 3; i++) {+ pollExtensionEventNext(chatty);+ }++ assertFalse(server.awaitExtensionsReady(300),+ "one extension polling repeatedly must not satisfy the other's slot");+ }++ /** A function with no extensions must not wait at all. */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_withNoExpectedExtensions_doesNotBlock() throws Exception {+ server.expectExtensions(0);++ long start = System.currentTimeMillis();+ assertTrue(server.awaitExtensionsReady(10_000));+ assertTrue(System.currentTimeMillis() - start < 1000, "zero extensions must not wait");+ }++ /** A never-ready extension must time out rather than block forever. */+ @Test+ @Timeout(30)+ void awaitExtensionsReady_timesOutWhenExtensionNeverPolls() throws Exception {+ server.expectExtensions(1);++ assertFalse(server.awaitExtensionsReady(300),+ "a missing first /event/next must report timeout, not hang");+ }++ /**+ * The behaviour abanna reported on PR #1773: "a later invoke can still be served after+ * init/error". A *new* invocation enqueued after the environment was condemned must not be+ * dispatched to the runtime — serving it would hide a failed adapter or security extension.+ * This is distinct from the in-flight invocation covered below, which must still complete.+ */+ @Test+ @Timeout(30)+ void invokeEnqueuedAfterExtensionFatalError_isNotServed() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad config\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertTrue(server.isFaulted());++ // A brand-new invocation arrives at the condemned environment.+ PendingInvocation later = new PendingInvocation(+ "req-after-fault", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ CompletableFuture<InvokeResult> laterFuture = server.enqueue(later);++ // It must be rejected outright rather than handed to the runtime's /next poller.+ InvokeResult result = laterFuture.get(5, TimeUnit.SECONDS);+ assertEquals("Unhandled", result.getFunctionError(),+ "an invoke enqueued after a fatal extension error must not be served");++ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .timeout(java.time.Duration.ofSeconds(5))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(204, next.statusCode(),+ "the runtime must not receive work from a condemned environment");+ }++ /**+ * A sibling extension already parked on /event/next when another reports a fatal error must be+ * released, not left hanging. The faulted guard in that handler means nothing will ever wake+ * it, so without an explicit drain the long-poll stays open until stop() eventually runs — up+ * to a full function timeout later. Harmless with one extension (the aws-lambda-web-adapter+ * case), but a second one hangs.+ *+ * <p>The response is the same 500 that a *new* poll receives after a fault; AWS specifies the+ * terminal state but not what an already-open long-poll gets, so this matches shipped+ * behaviour rather than inventing a second convention.+ */+ @Test+ @Timeout(30)+ void extensionFatalError_releasesSiblingExtensionParkedOnEventNext() throws Exception {+ String survivorId = registerExtension("surviving-extension", "INVOKE", "SHUTDOWN");+ String failingId = registerExtension("failing-extension", "INVOKE");++ // The survivor is parked on /event/next before the fault fires.+ CompletableFuture<HttpResponse<String>> parked = pollExtensionEventNext(survivorId);+ assertFalse(parked.isDone(), "sibling extension should be parked with no pending event");++ reportExtensionInitError(failingId);++ HttpResponse<String> response = parked.get(5, TimeUnit.SECONDS);+ assertEquals(500, response.statusCode(),+ "a parked sibling must be released when the environment is condemned");+ assertTrue(response.body().contains("Extension.SandboxFaulted"));+ }++ /**+ * AWS requires {@code Lambda-Extension-Function-Error-Type} on both error endpoints. Retiring+ * the environment is destructive and unrecoverable, so a report missing the required header+ * must be rejected *before* anything is condemned — otherwise a caller that got the contract+ * wrong silently kills a healthy container and the 202 tells them it worked.+ */+ @Test+ @Timeout(30)+ void extensionInitError_withoutErrorTypeHeader_isRejectedAndDoesNotFault() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(400, response.statusCode(),+ "a report missing the required error-type header must be rejected");+ assertFalse(server.isFaulted(),+ "a malformed report must not condemn the execution environment");+ }++ /** The same requirement on exit/error — both endpoints carry it in AWS. */+ @Test+ @Timeout(30)+ void extensionExitError_withoutErrorTypeHeader_isRejectedAndDoesNotFault() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(400, response.statusCode());+ assertFalse(server.isFaulted(),+ "a malformed report must not condemn the execution environment");+ }++ /**+ * An unknown identifier must be rejected rather than treated as a fatal report. This reverses+ * an earlier deliberate choice here (condemn regardless, on the theory that *some* extension+ * had failed): AWS validates the identifier first, and honouring an unrecognised one lets any+ * unauthenticated caller retire a healthy container.+ */+ @Test+ @Timeout(30)+ void extensionInitError_withUnknownIdentifier_isRejectedAndDoesNotFault() throws Exception {+ registerExtension("healthy-extension", "INVOKE");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", "not-a-real-id")+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(403, response.statusCode(),+ "an unknown identifier must be rejected, not honoured as a fatal report");+ assertFalse(server.isFaulted(),+ "an unknown extension must not condemn the execution environment");+ }++ /** A missing identifier is rejected the same way, and likewise must not condemn. */+ @Test+ @Timeout(30)+ void extensionInitError_withMissingIdentifier_isRejectedAndDoesNotFault() throws Exception {+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(403, response.statusCode());+ assertFalse(server.isFaulted());+ }++ /**+ * The positive path still works with both required headers present: the environment is+ * condemned exactly as before. Guards against the validation above over-rejecting.+ */+ @Test+ @Timeout(30)+ void extensionInitError_withRequiredHeaders_faultsAsBefore() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.BadConfig")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(202, response.statusCode());+ assertTrue(server.isFaulted(),+ "a well-formed report must still condemn the execution environment");+ }++ /**+ * AWS makes an init/exit error terminal for the Extensions API too, not only for runtime work:+ * once the environment is condemned no further extension call succeeds. A registration that+ * still returned 200 would hand a second extension an identifier that can never receive an+ * event, leaving it polling a container that is on its way to being retired.+ */+ @Test+ @Timeout(30)+ void extensionRegisterAfterFatalError_isRejected() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");+ reportExtensionInitError(extensionId);++ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", "late-extension")+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[\"INVOKE\"]}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(500, response.statusCode(),+ "registration into a condemned environment must fail, not hand back an identifier");+ assertTrue(response.body().contains("Extension.SandboxFaulted"));+ }++ /**+ * The same rule for /event/next, and the reason it is a 500 rather than the 204 used for an+ * orderly shutdown: an extension reads 204 as "nothing right now" and polls again, so a 204+ * here would spin it against a dead environment instead of telling it to stop.+ */+ @Test+ @Timeout(30)+ void extensionEventNextAfterFatalError_isRejected() throws Exception {+ String survivorId = registerExtension("surviving-extension", "INVOKE");+ String failingId = registerExtension("failing-extension", "INVOKE");+ reportExtensionInitError(failingId);++ // The still-registered extension polls after a *different* extension condemned the+ // environment — the fault is environment-wide, not scoped to the reporter.+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", survivorId)+ .timeout(java.time.Duration.ofSeconds(5))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ assertEquals(500, response.statusCode(),+ "a condemned environment must not park or serve further extension polls");+ assertTrue(response.body().contains("Extension.SandboxFaulted"));+ }++ /**+ * The in-flight invocation must still complete normally: real AWS condemns the environment for+ * *future* work, and tearing the runtime down mid-invoke would lose a result that did compute.+ */+ @Test+ @Timeout(30)+ void extensionFatalError_doesNotBreakInFlightInvocation() throws Exception {+ String extensionId = registerExtension("failing-extension", "INVOKE");++ PendingInvocation invocation = new PendingInvocation(+ "req-fatal", "{\"hello\":\"world\"}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ CompletableFuture<InvokeResult> resultFuture = server.enqueue(invocation);++ // Runtime picks the invocation up, then the extension dies mid-invoke.+ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, next.statusCode());++ httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"crashed\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertTrue(server.isFaulted());++ // The runtime's own response still lands and completes the invocation successfully.+ httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port+ + "/2018-06-01/runtime/invocation/req-fatal/response"))+ .POST(HttpRequest.BodyPublishers.ofString("{\"ok\":true}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());++ InvokeResult result = resultFuture.get(10, TimeUnit.SECONDS);+ assertEquals(200, result.getStatusCode());+ assertNull(result.getFunctionError(), "in-flight invocation must not be failed by the extension fault");+ assertEquals("{\"ok\":true}", new String(result.getPayload()));+ }++ /**+ * Regression for the historical orphaned-SHUTDOWN race (floci-io/floci#1882): stop()+ * could flip stopped=true and offer a SHUTDOWN event in the exact window a concurrent+ * /event/next request had already polled-empty and was about to check stopped, so the+ * request got a bare 204 with the SHUTDOWN never delivered. Stress it with many+ * iterations of a jittered race between stop() and /event/next, asserting SHUTDOWN is+ * always delivered exactly once — never orphaned (zero deliveries) and never duplicated.+ */+ @Test+ @Timeout(60)+ void extensionShutdown_racedAgainstStop_isNeverOrphaned() throws Exception {+ int iterations = 500;+ java.util.concurrent.atomic.AtomicInteger totalDelivered = new java.util.concurrent.atomic.AtomicInteger();++ for (int i = 0; i < iterations; i++) {+ // A fresh client per iteration — the port may be reused from an earlier iteration, and+ // a shared client's connection pool could otherwise hand back a stale pooled+ // connection to that iteration's now-closed server.+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();+ int freshPort = findFreePort();+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);+ freshServer.start().get(5, TimeUnit.SECONDS);+ try {+ String extensionId = registerExtensionOn(client, freshPort, "lambda-adapter", "SHUTDOWN");++ CompletableFuture<HttpResponse<String>> asyncNext = client.sendAsync(+ HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());++ // Give the request just enough time to actually connect and park server-side+ // (a few ms is plenty on localhost) before racing stop() against it — without+ // this, stop() can close the listening socket before the connection is even+ // established, which isn't the race we're testing.+ Thread.sleep(5);+ freshServer.stop();++ HttpResponse<String> response = asyncNext.get(5, TimeUnit.SECONDS);+ boolean deliveredHere = response.statusCode() == 200+ && "SHUTDOWN".equals(new JsonObject(response.body()).getString("eventType"));+ if (deliveredHere) {+ totalDelivered.incrementAndGet();+ } else {+ // Didn't land in this specific request — a subsequent poll with the same+ // identifier must still find the SHUTDOWN queued (never orphaned).+ HttpResponse<String> retry = client.send(HttpRequest.newBuilder()+ .uri(URI.create(+ "http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ if (retry.statusCode() == 200+ && "SHUTDOWN".equals(new JsonObject(retry.body()).getString("eventType"))) {+ totalDelivered.incrementAndGet();+ }+ }+ } finally {+ freshServer.stop().get(5, TimeUnit.SECONDS);+ // Each HttpClient owns a selector thread and connection pool; leaking one per+ // iteration across hundreds of iterations starves the JVM and makes even an+ // unrelated server's bind time out.+ client.close();+ }+ }++ assertEquals(iterations, totalDelivered.get(),+ "every iteration's SHUTDOWN must be delivered exactly once (never orphaned)");+ }++ /**+ * Equivalent race for the runtime invocation queue: races enqueue() against stop() with+ * jitter across many iterations, asserting the invocation's resultFuture always completes+ * (real dispatch or ContainerStopped) — a hang here trips the test timeout, an unambiguous+ * regression signal for an orphaned invocation.+ */+ @Test+ @Timeout(60)+ void enqueueRacedAgainstStop_alwaysCompletesResultFuture() throws Exception {+ int iterations = 500;+ java.util.concurrent.ThreadLocalRandom random = java.util.concurrent.ThreadLocalRandom.current();++ for (int i = 0; i < iterations; i++) {+ int freshPort = findFreePort();+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);+ freshServer.start().get(5, TimeUnit.SECONDS);+ try {+ PendingInvocation invocation = new PendingInvocation(+ "req-race-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());++ Thread stopper = new Thread(() -> {+ try {+ Thread.sleep(random.nextInt(0, 5));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ freshServer.stop();+ });+ stopper.start();+ freshServer.enqueue(invocation);+ stopper.join(5000);++ InvokeResult result = invocation.getResultFuture().get(5, TimeUnit.SECONDS);+ assertNotNull(result, "resultFuture must always complete, never hang");+ } finally {+ freshServer.stop().get(5, TimeUnit.SECONDS);+ }+ }+ }++ /**+ * /extension/register racing stop() (floci-io/floci#1882: register previously didn't+ * check stopped at all). Registration itself always completes before stop() can begin+ * (the race is purely over which one observes the other's effect first — stop() draining+ * an empty extensions map vs. register() finding stopped already true), and whatever+ * identifier comes back, a subsequent /event/next with it must eventually return SHUTDOWN+ * rather than hanging or 403ing inconsistently.+ */+ @Test+ @Timeout(60)+ void extensionRegister_racedAgainstStop_eventuallyDeliversShutdown() throws Exception {+ int iterations = 300;++ for (int i = 0; i < iterations; i++) {+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();+ int freshPort = findFreePort();+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);+ freshServer.start().get(5, TimeUnit.SECONDS);+ try {+ String extensionId = registerExtensionOn(client, freshPort, "lambda-adapter", "SHUTDOWN");+ // Race stop() against the extension's very first /event/next poll, which may+ // land before or after stop() — either way SHUTDOWN must still be delivered.+ CompletableFuture<HttpResponse<String>> asyncNext = client.sendAsync(+ HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ // Give the request time to actually connect and park before stop() closes+ // the listening socket — see extensionShutdown_racedAgainstStop_isNeverOrphaned.+ Thread.sleep(5);+ freshServer.stop();++ HttpResponse<String> response = asyncNext.get(5, TimeUnit.SECONDS);+ assertEquals(200, response.statusCode(),+ "a registered extension must eventually see SHUTDOWN, never hang/403");+ assertEquals("SHUTDOWN", new JsonObject(response.body()).getString("eventType"));+ } finally {+ freshServer.stop().get(5, TimeUnit.SECONDS);+ client.close();+ }+ }+ }++ /**+ * handleExtensionFatalError racing stop()'s SHUTDOWN fan-out: no exception, no+ * double-processing. The fatal-error POST completes before stop() begins tearing down+ * the connection (dispatched synchronously, unlike the long-polling /event/next above),+ * so the only race is stop()'s SHUTDOWN fan-out landing concurrently with the extension+ * having just been removed from the map.+ */+ @Test+ @Timeout(30)+ void extensionFatalError_racedAgainstStop_noExceptionNoDoubleProcessing() throws Exception {+ int iterations = 300;++ for (int i = 0; i < iterations; i++) {+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();+ int freshPort = findFreePort();+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);+ freshServer.start().get(5, TimeUnit.SECONDS);+ try {+ String extensionId = registerExtensionOn(client, freshPort, "failing-extension", "INVOKE", "SHUTDOWN");++ HttpResponse<String> response = client.send(HttpRequest.newBuilder()+ .uri(URI.create(+ "http://localhost:" + freshPort + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(202, response.statusCode(), "fatal-error endpoint must not throw under the race");++ // stop() runs immediately after — races its SHUTDOWN fan-out against the+ // extension having just been removed by the fatal-error handler above.+ freshServer.stop().get(5, TimeUnit.SECONDS);+ } finally {+ freshServer.stop().get(5, TimeUnit.SECONDS);+ client.close();+ }+ }+ }++ /**+ * Regression guard against reintroducing worker-thread pinning: NEXT_PATH must stay+ * non-blocking/event-loop-based, since the container's language-runtime bootstrap holds+ * this long-poll open for its entire idle lifetime. Parking more concurrent /next polls+ * than the default Quarkus worker pool (20 threads) must still succeed — this would fail+ * or hang if a future change moved NEXT_PATH to a blockingHandler/wait() design.+ */+ @Test+ @Timeout(30)+ void manyConcurrentNextPollers_exceedingWorkerPoolSize_allParkAndComplete() throws Exception {+ int serverCount = 30;+ List<RuntimeApiServer> servers = new ArrayList<>();+ List<Integer> ports = new ArrayList<>();+ try {+ for (int i = 0; i < serverCount; i++) {+ int freshPort = findFreePort();+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);+ freshServer.start().get(5, TimeUnit.SECONDS);+ servers.add(freshServer);+ ports.add(freshPort);+ }++ List<CompletableFuture<HttpResponse<String>>> pending = new ArrayList<>();+ for (int freshPort : ports) {+ pending.add(httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + freshPort + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString()));+ }++ Thread.sleep(500);+ for (CompletableFuture<HttpResponse<String>> f : pending) {+ assertFalse(f.isDone(), "all pollers should be parked, none held on a blocking thread");+ }++ for (int i = 0; i < serverCount; i++) {+ servers.get(i).enqueue(new PendingInvocation(+ "req-many-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>()));+ }++ for (CompletableFuture<HttpResponse<String>> f : pending) {+ assertEquals(200, f.get(5, TimeUnit.SECONDS).statusCode());+ }+ } finally {+ for (RuntimeApiServer s : servers) {+ s.stop().get(5, TimeUnit.SECONDS);+ }+ }+ }++ private String registerExtensionOn(HttpClient client, int targetPort, String name, String... events)+ throws Exception {+ String eventsJson = String.join(",", java.util.Arrays.stream(events).map(e -> "\"" + e + "\"").toList());+ HttpResponse<String> response = client.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + targetPort + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", name)+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[" + eventsJson + "]}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, response.statusCode());+ return response.headers().firstValue("Lambda-Extension-Identifier").orElseThrow();+ }++ /** Has the given extension report an init error, condemning the execution environment. */+ private void reportExtensionInitError(String extensionId) throws Exception {+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))+ .header("Lambda-Extension-Identifier", extensionId)+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad config\"}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(202, response.statusCode());+ assertTrue(server.isFaulted());+ }++ /**+ * Issues one /extension/event/next for the given extension, which is what marks it init-ready.+ *+ * <p>Deliberately async and un-awaited: with no event pending the handler parks the request+ * until an INVOKE or SHUTDOWN arrives, so a blocking send here would hang the test. The+ * readiness countdown happens as the handler runs, before it parks — callers observe it+ * through the barrier rather than through this response.+ *+ * <p>Sleeps briefly before returning so the request has actually reached the handler. Without+ * that, a caller asserting on the barrier immediately afterwards could observe the state from+ * before this poll and pass (or fail) for the wrong reason.+ */+ private CompletableFuture<HttpResponse<String>> pollExtensionEventNext(String extensionId)+ throws Exception {+ CompletableFuture<HttpResponse<String>> pending = httpClient.sendAsync(+ HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ Thread.sleep(150);+ return pending;+ }++ private String registerExtension(String name, String... events) throws Exception {+ String eventsJson = String.join(",", java.util.Arrays.stream(events).map(e -> "\"" + e + "\"").toList());+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))+ .header("Lambda-Extension-Name", name)+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[" + eventsJson + "]}"))+ .build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, response.statusCode());+ return response.headers().firstValue("Lambda-Extension-Identifier").orElseThrow();+ }++ /**+ * Lowest port used by {@link #findFreePort()}. Deliberately below the OS ephemeral range+ * (49152-65535 on macOS, 32768-60999 on typical Linux) — see {@link #findFreePort()}.+ */+ private static final int TEST_PORT_BASE = 20000;+ /**+ * Wide enough that the rotation does not wrap back onto a port still in TIME_WAIT: one run of+ * this class performs ~1600 port draws, and a closed listener's port stays unusable for tens of+ * seconds afterwards. Running several test classes in the same JVM multiplies that count, which+ * is when a narrower window started producing sporadic connect failures.+ */+ private static final int TEST_PORT_RANGE = 25000;++ /** Rotates across the range so consecutive calls don't retry a port still in TIME_WAIT. */+ private static final java.util.concurrent.atomic.AtomicInteger NEXT_PORT_OFFSET =+ new java.util.concurrent.atomic.AtomicInteger(+ java.util.concurrent.ThreadLocalRandom.current().nextInt(TEST_PORT_RANGE));++ /**+ * Returns a port that is free <em>and</em> outside the OS ephemeral range.+ *+ * <p>Binding port 0 (the obvious implementation) draws from the ephemeral range, which is+ * exactly where the OS and background applications allocate their own listeners. Long-running+ * desktop agents routinely hold stable ports in that range — on the machine this was diagnosed+ * on, {@code LogiPlugin} (49200-49204), {@code rapportd} (56698, 63091-63092) and+ * {@code CommCenter} (65000-65001) all sit inside it. The stress tests below perform well over+ * a thousand port draws per run, so a collision is near-certain: the probe socket closes, the+ * other process (or the OS) claims the port before Vert.x binds it, and the test's request is+ * answered by a stranger — observed as a spurious {@code 501} from an unrelated HTTP daemon, or+ * as a bind failure/start() timeout. Neither has anything to do with the concurrency under test.+ *+ * <p>Ports 20000-29999 sit below every common ephemeral range, so nothing else on the host is+ * handing them out. The bind check still guards against a genuinely occupied port.+ */private static int findFreePort() throws IOException {- try (ServerSocket socket = new ServerSocket(0)) {- return socket.getLocalPort();+ for (int attempt = 0; attempt < TEST_PORT_RANGE; attempt++) {+ int candidate = TEST_PORT_BASE+ + Math.floorMod(NEXT_PORT_OFFSET.getAndIncrement(), TEST_PORT_RANGE);+ try (ServerSocket socket = new ServerSocket()) {+ // Without SO_REUSEADDR the probe leaves the port in TIME_WAIT, which would block+ // the server that is about to bind it for real.+ socket.setReuseAddress(true);+ socket.bind(new InetSocketAddress("0.0.0.0", candidate));+ return candidate;+ } catch (IOException inUse) {+ // Occupied — try the next port in the rotation.+ }}+ throw new IOException("no free port in " + TEST_PORT_BASE + "-" + (TEST_PORT_BASE + TEST_PORT_RANGE));}}