Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): SIGTERM container before closing runtime API socket (#2098)
Guinevere Friedmanb911ccd01392
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 3e1ad97f85..7eb9606a55 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -162,11 +162,23 @@ public class ContainerLifecycleManager {* @param logStream optional log stream to close (may be null)*/public void stopAndRemove(String containerId, Closeable logStream) {+ stopAndRemove(containerId, logStream, 5);+ }++ /**+ * Stops and removes a container, closing any associated log stream.+ *+ * @param containerId the container ID to stop and remove+ * @param logStream optional log stream to close (may be null)+ * @param stopTimeoutSeconds seconds to wait for the container to stop cleanly after+ * SIGTERM before Docker sends SIGKILL. Must be a non-negative integer.+ */+ public void stopAndRemove(String containerId, Closeable logStream, int stopTimeoutSeconds) {LOG.infov("Stopping container {0}", containerId);boolean stoppedOrMissing = false;try {- dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();+ dockerClient.stopContainerCmd(containerId).withTimeout(stopTimeoutSeconds).exec();stoppedOrMissing = true;} catch (NotFoundException e) {LOG.debugv("Container {0} not found (already removed)", containerId);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 94df0c6596..9cba571cec 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -445,17 +445,43 @@ public class ContainerLauncher {LOG.infov("Stopping container {0}", handle.getContainerId());handle.setState(ContainerState.STOPPED);+ RuntimeApiServer server = handle.getRuntimeApiServer();++ // Quiesce first: stop accepting new work, notify extensions, complete pending+ // invocations. Existing runtime pollers on /invocation/next stay parked.+ server.quiesce();++ // Then SIGTERM the container. The runtime library's default SIGTERM handler exits+ // the process while the runtime API socket is still up, matching real AWS Lambda's+ // shutdown flow.+ //+ // stopTimeoutSeconds is the ceiling docker waits for PID 1 to exit before SIGKILL,+ // not a grace floor: the runtime typically exits promptly, so `docker stop`+ // returns shortly after and this value rarely matters end-to-end. Sized against+ // AWS's documented Shutdown-phase tiers as a defensive upper bound only — 2s+ // when at least one extension is registered (matching AWS's 2000ms+ // external-extensions budget), else 1s. NOTE: extensions run as sibling+ // `docker exec`s (not children of PID 1) and are killed when the container+ // exits; the 2s ceiling does not provide a guaranteed grace window for+ // extension shutdown work — only that we will not preempt PID 1 for at least+ // that long if it happens to be slow.+ int stopTimeoutSeconds = server.hasRegisteredExtensions() ? 2 : 1;+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream(),+ stopTimeoutSeconds);++ // Only now close the runtime API socket. Any pollers that were still parked when+ // the container exited get terminated by their end of the connection closing;+ // this call releases the port for reuse.try {- handle.getRuntimeApiServer().stop().get(5, TimeUnit.SECONDS);+ server.close().get(5, TimeUnit.SECONDS);} catch (InterruptedException e) {Thread.currentThread().interrupt();} catch (ExecutionException | TimeoutException e) {LOG.warnv(e, "RuntimeApiServer did not close cleanly for container {0}",handle.getContainerId());} finally {- runtimeApiServerFactory.release(handle.getRuntimeApiServer());+ runtimeApiServerFactory.release(server);}- lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());}/**diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.javaindex 49b4aa33e4..7349a463b4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java@@ -127,6 +127,72 @@ public class RuntimeApiServer {private boolean stopped;private CompletableFuture<Void> closeFuture;+ // Set by quiesce() (under lock); completes when every parked extension SHUTDOWN+ // has flushed. close() awaits this before dropping the httpServer (see #2142).+ // Null if quiesce() has not run.+ private CompletableFuture<Void> extensionWritesFlushed;++ /** Test-only: number of /next pollers parked in waitingContexts, for+ * tests that want to await the parked state deterministically without sleeping. */+ int waitingContextsSize() {+ synchronized (lock) {+ return waitingContexts.size();+ }+ }++ /** Test-only: number of invocations queued waiting for a /next poller. Used by+ * the requeue-on-write-failure test to verify the requeue actually happened. */+ int pendingQueueSize() {+ synchronized (lock) {+ return pendingQueue.size();+ }+ }++ /** Test-only: number of invocations currently in flight. Used alongside+ * {@link #pendingQueueSize()} to verify a requeue-on-failure both offered+ * to pendingQueue AND cleared the stale inFlight entry. */+ int inFlightSize() {+ return inFlight.size();+ }++ /**+ * Test-only: called at the start of enqueue()'s deferred sendInvocation,+ * before the guard's `stopped` re-check. A test subclass overrides this to+ * gate the dispatch on a CountDownLatch and drive a specific interleaving+ * with quiesce(). Empty no-op in production.+ */+ protected void beforeEnqueueDeferredDispatch() { }++ /**+ * Test-only: called at the start of NEXT_PATH's post-lock guard block, before+ * the guard's `stopped` re-check. Same shape as {@link #beforeEnqueueDeferredDispatch}+ * but for the synchronous handler-thread dispatch path.+ */+ protected void beforeNextPathDispatchGuard() { }++ /**+ * Test-only: called inside quiesce() immediately after the lock has been+ * released, before any outside-lock work runs. Test subclasses override+ * to freeze quiesce at that point and drive a dispatch through the guard+ * against a supposedly-partially-swept state, proving atomicity holds.+ */+ protected void afterQuiesceStoppedFlagSet() { }++ /**+ * Test-only: called inside enqueue()'s deferred dispatch after lock release,+ * before the send/requeue decision is acted on. Test subclasses override to+ * mutate the parked ctx (e.g. end it) and prove the decision was committed+ * atomically inside the lock.+ */+ protected void afterEnqueueDispatchLockReleased(RoutingContext waitingCtx) { }++ /**+ * Test-only: called at the top of sendInvocation so tests can observe whether+ * a dispatch actually ran. Distinguishes "sent to a dead ctx" from "skipped,+ * stranded" — both leave the invocation in inFlight otherwise.+ */+ protected void beforeSendInvocationWrite(String requestId) { }+// Set once an extension reports an init/exit error. Real AWS treats both as fatal to the// execution environment, so the container must neither accept new work nor be reused.//@@ -183,6 +249,18 @@ public class RuntimeApiServer {return port;}+ /**+ * True if at least one extension has completed registration via+ * {@code /extension/register}. Callers use this to pick the Shutdown-phase grace tier+ * (see AWS's documented 0/500/2000ms limits): a container with a registered extension+ * gets the 2s external-extensions budget, otherwise the runtime-only sub-budget.+ */+ public boolean hasRegisteredExtensions() {+ synchronized (lock) {+ return !extensions.isEmpty();+ }+ }+/*** True once a registered extension reported an init or exit error. The execution environment* is condemned: {@code WarmPool} must not return this container to the pool or hand it out@@ -252,25 +330,46 @@ public class RuntimeApiServer {// exhaustion when many warm containers poll concurrently.router.get(NEXT_PATH).handler(ctx -> {PendingInvocation toDispatch = null;- boolean send204 = false;+ boolean faultedNow = false;synchronized (lock) {- // `faulted` alongside `stopped`: a condemned environment must not be handed work,- // including an invocation that was queued just before the fault was reported.- if (stopped || faulted) {- send204 = true;+ if (faulted) {+ // Faulted environment: signal the fault so the runtime exits cleanly rather+ // than parking forever. Distinct from `stopped` — stopped is a normal+ // teardown, faulted means something invariant-breaking happened upstream.+ faultedNow = true;+ } else if (stopped) {+ // Teardown in progress. Park the poll; the container process is being+ // SIGTERMed and will exit before the socket closes, matching how real+ // AWS Lambda tears an environment down.+ waitingContexts.add(ctx);} else {toDispatch = pendingQueue.poll();if (toDispatch == null) {waitingContexts.add(ctx);+ } else {+ // Move to inFlight under the same lock so a concurrent quiesce()+ // sweep sees it — otherwise the invocation escapes both queues+ // and its future is never completed.+ inFlight.put(toDispatch.getRequestId(), toDispatch);}}}- if (send204) {+ if (faultedNow) {ctx.response().setStatusCode(204).end();} else if (toDispatch != null) {- sendInvocation(ctx, toDispatch);+ beforeNextPathDispatchGuard();+ // Re-check `stopped` under the lock: quiesce() may have run since our+ // poll above, in which case it completed toDispatch's future with+ // ContainerStopped and dispatching now would silently discard /response.+ boolean alreadyHandled;+ synchronized (lock) {+ alreadyHandled = stopped;+ }+ if (!alreadyHandled) {+ sendInvocation(ctx, toDispatch);+ }}- // else: parked — enqueue() or stop() will dispatch this ctx later.+ // else: parked — enqueue() or the httpServer close will dispatch/end this ctx.});// POST /runtime/invocation/{requestId}/response — success@@ -479,22 +578,31 @@ public class RuntimeApiServer {});}- public CompletableFuture<Void> stop() {- CompletableFuture<Void> closed;- List<RoutingContext> contextsToClose204;- List<Supplier<Future<Void>>> dispatches = new ArrayList<>();+ /**+ * Mark the server stopped and settle all bookkeeping, without closing the underlying HTTP+ * server. Idempotent — repeated calls are no-ops.+ *+ * Runtime pollers already parked on {@code /invocation/next} stay parked; the caller is+ * expected to shut the container process down (docker stop → SIGTERM) before invoking+ * {@link #close()}, so those pollers are terminated by the container exit rather than by+ * a server-side response. Extensions parked on {@code /event/next} get a {@code Shutdown}+ * event (this is documented AWS behavior for the Extensions API). Pending and in-flight+ * invocations are completed with {@code ContainerStopped}.+ *+ * Call {@link #close()} afterward to release the port.+ */+ public void quiesce() {+ List<Supplier<Future<Void>>> extensionDispatches = new ArrayList<>();List<PendingInvocation> invocationsToFailContainerStopped;+ List<PendingInvocation> inFlightToFail;+ CompletableFuture<Void> writesFuture = new CompletableFuture<>();synchronized (lock) {- if (closeFuture != null) {- return closeFuture;+ if (stopped) {+ return;}stopped = true;- closed = new CompletableFuture<>();- closeFuture = closed;-- contextsToClose204 = new ArrayList<>(waitingContexts);- waitingContexts.clear();+ extensionWritesFlushed = writesFuture;ExtensionEvent shutdownEvent = ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN");for (RegisteredExtension ext : extensions.values()) {@@ -503,7 +611,7 @@ public class RuntimeApiServer {}RoutingContext waitingCtx = ext.takeWaitingContext();if (waitingCtx != null) {- dispatches.add(() -> {+ extensionDispatches.add(() -> {if (waitingCtx.response().ended()) {return Future.succeededFuture();}@@ -516,6 +624,75 @@ public class RuntimeApiServer {invocationsToFailContainerStopped = new ArrayList<>(pendingQueue);pendingQueue.clear();+ // Snapshot and clear inFlight inside the lock, atomic with stopped=true.+ // The dispatch guards in NEXT_PATH and enqueue()'s deferred callback rely on+ // this: once quiesce releases the lock, stopped=true implies inFlight is+ // empty, so a guard reading `stopped` cannot deliver an invocation whose+ // future quiesce is about to complete with ContainerStopped.+ inFlightToFail = new ArrayList<>(inFlight.values());+ inFlight.clear();+ }+ afterQuiesceStoppedFlagSet();++ // Extension SHUTDOWN writes go through the event loop; chain on end() rather+ // than the scheduled handler's return, because response.end() is async and the+ // bytes are not on the wire until its future completes. close() awaits+ // extensionWritesFlushed before dropping the httpServer (see #2142).+ //+ // Runtime pollers stay parked — no 204. They are terminated either by the+ // container process exiting on SIGTERM (the ContainerLauncher path, which+ // interleaves docker stop between quiesce and close) or by the httpServer+ // close itself (the stop() wrapper path); either matches real AWS Lambda's+ // teardown better than an undocumented 204 on /invocation/next would.+ if (extensionDispatches.isEmpty()) {+ writesFuture.complete(null);+ } else {+ AtomicInteger remainingWrites = new AtomicInteger(extensionDispatches.size());+ for (Supplier<Future<Void>> dispatch : extensionDispatches) {+ vertx.runOnContext(v -> dispatch.get().onComplete(ar -> {+ if (remainingWrites.decrementAndGet() == 0) {+ writesFuture.complete(null);+ }+ }));+ }+ }++ // Fail queued invocations that were never consumed by /next.+ for (PendingInvocation pending : invocationsToFailContainerStopped) {+ pending.getResultFuture().complete(+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));+ }++ // Complete any in-flight invocations with error.+ for (PendingInvocation inv : inFlightToFail) {+ inv.getResultFuture().complete(+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, inv.getRequestId()));+ }+ }++ /**+ * Close the underlying HTTP server, releasing the bound port. Idempotent — the returned+ * future resolves once the server has actually closed (first call) or immediately with+ * the same future (subsequent calls). Any parked runtime pollers are terminated by the+ * underlying channel closing.+ *+ * {@link #quiesce()} should be called first so that any registered extensions have the+ * chance to receive a Shutdown event before their sockets die.+ */+ public CompletableFuture<Void> close() {+ CompletableFuture<Void> closed;+ CompletableFuture<Void> awaitWrites;+ synchronized (lock) {+ if (closeFuture != null) {+ return closeFuture;+ }+ closed = new CompletableFuture<>();+ closeFuture = closed;+ // Await any extension SHUTDOWN writes quiesce() scheduled before closing+ // the httpServer, so their responses don't race the socket teardown.+ awaitWrites = extensionWritesFlushed != null+ ? extensionWritesFlushed+ : CompletableFuture.completedFuture(null);}Runnable closeServer = () -> {@@ -533,54 +710,21 @@ public class RuntimeApiServer {closed.complete(null);}};-- // Wake any parked /next pollers with 204 (container shutting down — runtime will exit) and- // dispatch SHUTDOWN to every extension already parked on /event/next, then close the HTTP- // server. Each write is scheduled on the event loop via vertx.runOnContext(...), and the- // server is closed only once every write's response has actually been flushed — tracked by- // remainingWrites, decremented from each end() future's completion rather than when the- // scheduled handler returns (response.end() is asynchronous, so the bytes are not on the- // wire yet when the handler returns). So a parked extension's SHUTDOWN — or a poller's 204- // — is fully written before its connection is torn down, instead of racing- // httpServer.close() and orphaning the extension with an EOF. See issue #2142.- int parkedWrites = contextsToClose204.size() + dispatches.size();- if (parkedWrites == 0) {- closeServer.run();- } else {- AtomicInteger remainingWrites = new AtomicInteger(parkedWrites);- Runnable afterWrite = () -> {- if (remainingWrites.decrementAndGet() == 0) {- closeServer.run();- }- };- for (RoutingContext ctx : contextsToClose204) {- vertx.runOnContext(v -> {- Future<Void> written = ctx.response().ended()- ? Future.succeededFuture()- : ctx.response().setStatusCode(204).end();- written.onComplete(ar -> afterWrite.run());- });- }- for (Supplier<Future<Void>> dispatch : dispatches) {- vertx.runOnContext(v -> dispatch.get().onComplete(ar -> afterWrite.run()));- }- }-- // Fail queued invocations that were never consumed by /next.- for (PendingInvocation pending : invocationsToFailContainerStopped) {- pending.getResultFuture().complete(- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));- }-- // Complete any in-flight invocations with error.- inFlight.values().forEach(inv ->- inv.getResultFuture().complete(- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, inv.getRequestId())));- inFlight.clear();-+ awaitWrites.whenComplete((v, err) -> closeServer.run());return closed;}+ /**+ * Convenience wrapper: {@link #quiesce()} then {@link #close()} back-to-back, matching the+ * pre-split single-step teardown. Callers that need to SIGTERM the container between the+ * quiesce and close phases (which is the AWS-faithful teardown) should call the two+ * methods directly rather than this wrapper.+ */+ public CompletableFuture<Void> stop() {+ quiesce();+ return close();+ }+public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {boolean rejected;List<Runnable> dispatches = new ArrayList<>();@@ -621,6 +765,11 @@ public class RuntimeApiServer {waitingCtxForInvocation = waitingContexts.poll();if (waitingCtxForInvocation == null) {pendingQueue.offer(invocation);+ } else {+ // Move to inFlight under the same lock so a concurrent quiesce()+ // sweep sees it — otherwise the invocation escapes both queues and+ // the caller hangs until the function deadline.+ inFlight.put(invocation.getRequestId(), invocation);}}}@@ -638,14 +787,39 @@ public class RuntimeApiServer {if (waitingCtxForInvocation != null) {final RoutingContext waitingCtx = waitingCtxForInvocation;vertx.runOnContext(v -> {- if (!waitingCtx.response().ended()) {- sendInvocation(waitingCtx, invocation);- } else {- // Connection closed between park and dispatch — re-queue.- synchronized (lock) {+ beforeEnqueueDeferredDispatch();+ // Two things could have changed between enqueue()'s lock release and+ // this deferred dispatch reaching the event loop:+ // 1. quiesce() may have run — atomic with clearing inFlight and+ // completing the caller's future with ContainerStopped. Skip.+ // 2. The parked client may have disconnected — the ctx is ended and+ // writing to it would fail. Requeue for the next /next poller+ // and drop this stale inFlight entry (the next poller will re-put+ // the invocation into inFlight under its own lock).+ //+ // Read waitingCtx.response().ended() ONCE, under the lock, so the+ // dispatch/requeue decision is committed atomically. Reading it a+ // second time outside the lock (as the code did previously) let a+ // disconnect between the two reads produce a stranded inFlight entry:+ // the inside-lock read saw !ended → no requeue; the outside-lock read+ // saw ended → no send; neither branch fired.+ boolean shouldDispatch;+ synchronized (lock) {+ if (stopped) {+ return;+ }+ if (waitingCtx.response().ended()) {pendingQueue.offer(invocation);+ inFlight.remove(invocation.getRequestId());+ shouldDispatch = false;+ } else {+ shouldDispatch = true;}}+ afterEnqueueDispatchLockReleased(waitingCtx);+ if (shouldDispatch) {+ sendInvocation(waitingCtx, invocation);+ }});}return invocation.getResultFuture();@@ -677,19 +851,40 @@ public class RuntimeApiServer {}private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {- inFlight.put(invocation.getRequestId(), invocation);-+ beforeSendInvocationWrite(invocation.getRequestId());+ // Invariant: callers put the invocation in inFlight under the lock before+ // dispatching, so no inFlight.put here.byte[] payload = invocation.getPayload();String body = (payload != null && payload.length > 0)? new String(payload): "{}";- ctx.response()- .setStatusCode(200)- .putHeader("Content-Type", "application/json")- .putHeader("Lambda-Runtime-Aws-Request-Id", invocation.getRequestId())- .putHeader("Lambda-Runtime-Invoked-Function-Arn", invocation.getFunctionArn())- .putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs()))- .end(body);+ // Vert.x reports a client-side disconnect two ways: setStatusCode/putHeader+ // throw synchronously if the response is already ended, .end() returns a+ // failed Future if the socket dies mid-flush. Funnel both into one handler.+ Future<Void> write;+ try {+ write = ctx.response()+ .setStatusCode(200)+ .putHeader("Content-Type", "application/json")+ .putHeader("Lambda-Runtime-Aws-Request-Id", invocation.getRequestId())+ .putHeader("Lambda-Runtime-Invoked-Function-Arn", invocation.getFunctionArn())+ .putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs()))+ .end(body);+ } catch (IllegalStateException alreadyEnded) {+ write = Future.failedFuture(alreadyEnded);+ }+ write.onFailure(err -> {+ // Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.+ // Skip if quiesce() beat us — it already swept inFlight and settled the+ // future with ContainerStopped.+ synchronized (lock) {+ if (stopped) {+ return;+ }+ pendingQueue.offer(invocation);+ inFlight.remove(invocation.getRequestId());+ }+ });}private Future<Void> sendExtensionEvent(RoutingContext ctx, ExtensionEvent event) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.javaindex 8cb210c951..01e76002cc 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java@@ -20,6 +20,7 @@ import java.net.http.HttpResponse;import java.util.ArrayList;import java.util.List;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CountDownLatch;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;@@ -67,6 +68,22 @@ class RuntimeApiServerTest {vertx.close().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);}+ /**+ * Polls until at least {@code count} runtime pollers have parked in the server's+ * {@code waitingContexts}. Replaces Thread.sleep-based waits so tests don't guess+ * how long the client-side TCP connect + server-side park sequence takes.+ */+ private void awaitWaitingContexts(int count) throws InterruptedException {+ long deadline = System.currentTimeMillis() + 5000;+ while (System.currentTimeMillis() < deadline) {+ if (server.waitingContextsSize() >= count) return;+ Thread.sleep(10);+ }+ throw new AssertionError(+ "expected at least " + count + " parked /next poller(s); got "+ + server.waitingContextsSize());+ }+@Test@Timeout(15)void nextEndpoint_blocksUntilInvocationArrives() throws Exception {@@ -245,9 +262,325 @@ class RuntimeApiServerTest {assertTrue(payload.contains("ContainerStopped"));}+ /**+ * Replaces the default {@code server} with a subclass whose test-only overrides+ * gate the four race points on the caller-supplied Runnables. Used only by the+ * race tests below; other tests keep the plain server. Rebinds the port because+ * start() binds on construction.+ */+ private void installGatedServer(Runnable beforeEnqueueDispatch,+ Runnable beforeNextPathGuard,+ Runnable afterQuiesceStoppedFlag) throws Exception {+ server.stop().get(5, TimeUnit.SECONDS);+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port) {+ @Override protected void beforeEnqueueDeferredDispatch() { beforeEnqueueDispatch.run(); }+ @Override protected void beforeNextPathDispatchGuard() { beforeNextPathGuard.run(); }+ @Override protected void afterQuiesceStoppedFlagSet() { afterQuiesceStoppedFlag.run(); }+ };+ server.start().get(5, TimeUnit.SECONDS);+ }++ /**+ * Variant of {@link #installGatedServer} for the ctx-ends-during-dispatch race:+ * inject state after enqueue()'s dispatch lock is released, observe whether+ * sendInvocation subsequently ran.+ */+ private void installGatedServerWithDispatchObservation(+ java.util.function.Consumer<io.vertx.ext.web.RoutingContext> afterEnqueueLockReleased,+ java.util.function.Consumer<String> onSendInvocation) throws Exception {+ server.stop().get(5, TimeUnit.SECONDS);+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port) {+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {+ afterEnqueueLockReleased.accept(waitingCtx);+ }+ @Override protected void beforeSendInvocationWrite(String requestId) {+ onSendInvocation.accept(requestId);+ }+ };+ server.start().get(5, TimeUnit.SECONDS);+ }+@Test@Timeout(15)- void stopWakesParkedPollerImmediately() throws Exception {+ void enqueueDeferredDispatch_racedByQuiesce_doesNotDeliverAfterSweep() throws Exception {+ // The dispatch guard is only sound if quiesce()'s inFlight sweep is atomic+ // with stopped=true under the lock. Prove that by driving the exact+ // interleaving the pre-hampsterx-fix code exposed. Sequence:+ //+ // 1. Park a /next poller server-side; enqueue an invocation → puts inv into+ // inFlight under the lock, schedules a deferred sendInvocation on the+ // event loop.+ // 2. The deferred dispatch fires but blocks on `holdDispatch` before it+ // can reach the guard's stopped-recheck.+ // 3. Run quiesce() on a background thread. Sets stopped=true, sweeps+ // inFlight (atomically, under fix) or defers it (pre-fix), releases+ // lock. Blocks at afterQuiesceStoppedFlagSet.+ // 4. Release `holdDispatch`. Guard reads `stopped`: under the fix, sees+ // atomically-cleared state and skips. Under the pre-fix code, sweep+ // hasn't run outside the lock yet — old guard `stopped && inFlight+ // .get(id)==null` would be false and dispatch would deliver.+ // 5. Release quiesce.+ //+ // Assertion: parked client MUST NOT have received a 200 with our request-id.+ CountDownLatch dispatchEntered = new CountDownLatch(1);+ CountDownLatch holdDispatch = new CountDownLatch(1);+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);+ CountDownLatch releaseQuiesce = new CountDownLatch(1);+ installGatedServer(+ () -> {+ dispatchEntered.countDown();+ try { holdDispatch.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ },+ () -> { /* NEXT_PATH not exercised here */ },+ () -> {+ quiesceReachedHook.countDown();+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ });++ HttpRequest parkedRequest = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build();+ CompletableFuture<HttpResponse<String>> parkedResponse =+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ PendingInvocation invocation = new PendingInvocation(+ "req-latch-enqueue", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);+ assertTrue(dispatchEntered.await(5, TimeUnit.SECONDS),+ "deferred dispatch should have entered the pre-guard hook");++ // Kick off quiesce on a background thread; it will freeze at the post-lock hook.+ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(() -> server.quiesce());+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),+ "quiesce should have released its lock and reached the post-lock hook");++ // NOW release the dispatch while quiesce is frozen post-lock — the exact window+ // the pre-fix code was vulnerable in. Guard's `stopped=true` under fix must+ // atomically imply inFlight empty.+ holdDispatch.countDown();+ // Give the guard time to run (blocking on releaseQuiesce first).+ Thread.sleep(200);+ releaseQuiesce.countDown();+ quiesceDone.get(5, TimeUnit.SECONDS);++ try {+ HttpResponse<String> response = parkedResponse.get(2, TimeUnit.SECONDS);+ assertFalse(response.statusCode() == 200+ && "req-latch-enqueue".equals(response.headers()+ .firstValue("Lambda-Runtime-Aws-Request-Id").orElse("")),+ "guard failed: parked poller received the invocation after quiesce; "+ + "would cause silent discard of /response. Got "+ + response.statusCode());+ } catch (Exception expected) {+ // socket close from tearDown — expected shape.+ }+ InvokeResult result = invocation.getResultFuture().get(2, TimeUnit.SECONDS);+ assertEquals("Unhandled", result.getFunctionError());+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));+ }++ @Test+ @Timeout(15)+ void nextPathDispatch_racedByQuiesce_doesNotDeliverAfterSweep() throws Exception {+ // Symmetric to the enqueue race but on NEXT_PATH: an invocation sits in+ // pendingQueue. The handler polls it out under the lock, moves to inFlight,+ // releases lock, then reaches the guard. We hold the guard, run quiesce on+ // a background thread to freeze it post-lock, release the guard.+ CountDownLatch dispatchEntered = new CountDownLatch(1);+ CountDownLatch holdDispatch = new CountDownLatch(1);+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);+ CountDownLatch releaseQuiesce = new CountDownLatch(1);+ installGatedServer(+ () -> { /* enqueue path not exercised here */ },+ () -> {+ dispatchEntered.countDown();+ try { holdDispatch.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ },+ () -> {+ quiesceReachedHook.countDown();+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ });++ PendingInvocation invocation = new PendingInvocation(+ "req-latch-nextpath", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ HttpRequest nextRequest = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build();+ CompletableFuture<HttpResponse<String>> nextResponse =+ httpClient.sendAsync(nextRequest, HttpResponse.BodyHandlers.ofString());+ assertTrue(dispatchEntered.await(5, TimeUnit.SECONDS),+ "NEXT_PATH handler should have entered the pre-guard hook");++ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(() -> server.quiesce());+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),+ "quiesce should have released its lock and reached the post-lock hook");++ holdDispatch.countDown();+ Thread.sleep(200);+ releaseQuiesce.countDown();+ quiesceDone.get(5, TimeUnit.SECONDS);++ try {+ HttpResponse<String> response = nextResponse.get(2, TimeUnit.SECONDS);+ assertFalse(response.statusCode() == 200+ && "req-latch-nextpath".equals(response.headers()+ .firstValue("Lambda-Runtime-Aws-Request-Id").orElse("")),+ "guard failed: /next received the invocation after quiesce; "+ + "would cause silent discard of /response. Got "+ + response.statusCode());+ } catch (Exception expected) {+ // socket close from tearDown — expected shape.+ }+ InvokeResult result = invocation.getResultFuture().get(2, TimeUnit.SECONDS);+ assertEquals("Unhandled", result.getFunctionError());+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));+ }++ @Test+ @Timeout(15)+ void enqueueDeferredDispatch_ctxEndsAfterLockRelease_dispatchesOrRequeuesAtomically() throws Exception {+ // Pre-fix, enqueue()'s deferred callback read waitingCtx.response().ended()+ // twice — once inside the lock deciding requeue, once outside deciding send.+ // A disconnect between the two reads made neither branch fire and stranded+ // the invocation in inFlight until the function deadline. Force that+ // interleaving by ending the ctx from afterEnqueueDispatchLockReleased.+ java.util.concurrent.atomic.AtomicInteger sendInvocationCalls =+ new java.util.concurrent.atomic.AtomicInteger();+ installGatedServerWithDispatchObservation(+ waitingCtx -> {+ if (!waitingCtx.response().ended()) {+ waitingCtx.response().setStatusCode(500).end();+ }+ },+ requestId -> sendInvocationCalls.incrementAndGet());++ HttpRequest parkedRequest = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build();+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ PendingInvocation invocation = new PendingInvocation(+ "req-ctx-ends", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ // Wait for the runOnContext-scheduled dispatch to have fired.+ Thread.sleep(500);++ assertEquals(1, sendInvocationCalls.get(),+ "sendInvocation must run exactly once — a zero count means the "+ + "invocation was stranded (the pre-fix stranded-inFlight bug).");+ }++ @Test+ @Timeout(15)+ void sendInvocation_writeFails_requeuesAndClearsInFlight() throws Exception {+ // Post-atomic-decision race: disconnect between dispatch commitment and+ // .end() flush strands the invocation in inFlight. Force it by ending the+ // parked ctx from beforeSendInvocationWrite; assert onFailure requeues ++ // clears inFlight.+ java.util.concurrent.atomic.AtomicReference<io.vertx.ext.web.RoutingContext> parkedCtx =+ new java.util.concurrent.atomic.AtomicReference<>();+ server.stop().get(5, TimeUnit.SECONDS);+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port) {+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {+ parkedCtx.set(waitingCtx);+ }+ @Override protected void beforeSendInvocationWrite(String requestId) {+ io.vertx.ext.web.RoutingContext ctx = parkedCtx.get();+ if (ctx != null && !ctx.response().ended()) {+ ctx.response().setStatusCode(500).end();+ }+ }+ };+ server.start().get(5, TimeUnit.SECONDS);++ HttpRequest parkedRequest = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build();+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ PendingInvocation invocation = new PendingInvocation(+ "req-write-fails", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ // Wait for the runOnContext-scheduled dispatch (and its failed write) to fire.+ Thread.sleep(500);++ assertEquals(1, server.pendingQueueSize(),+ "invocation must be requeued after write failure — zero means stranded.");+ assertEquals(0, server.inFlightSize(),+ "inFlight must be cleared alongside the requeue.");+ }++ @Test+ @Timeout(15)+ void quiesceAtomicallyClearsInFlightWithStopped() throws Exception {+ // The dispatch guards in enqueue()'s deferred callback and NEXT_PATH read+ // `stopped` under the server lock and act on it. Their correctness depends on+ // an invariant: once quiesce() releases its lock, `stopped=true` implies+ // inFlight is empty. If quiesce() cleared inFlight *outside* the lock, a+ // dispatch could observe stopped=true while an inFlight entry it's about to+ // deliver was still present — the guard would let the delivery through,+ // quiesce would then complete the future with ContainerStopped, and the+ // runtime's /response would be silently discarded.+ //+ // Put entries in inFlight via full enqueue → /next round trips (matches the+ // real code path), then call quiesce and assert all their futures completed.+ List<PendingInvocation> invocations = new ArrayList<>();+ for (int i = 0; i < 5; i++) {+ PendingInvocation inv = new PendingInvocation(+ "req-atomic-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(inv);+ invocations.add(inv);+ HttpResponse<String> resp = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port+ + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, resp.statusCode());+ }++ server.quiesce();++ // quiesce() sets stopped=true AND clears inFlight in the same synchronized+ // block, so both observations are atomic. Every invocation must be completed.+ for (PendingInvocation inv : invocations) {+ InvokeResult result = inv.getResultFuture().get(2, TimeUnit.SECONDS);+ assertEquals("Unhandled", result.getFunctionError(),+ "invocation " + inv.getRequestId() + " must be completed by quiesce");+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));+ }+ }++ @Test+ @Timeout(15)+ void closeTerminatesParkedPollerWithoutResponse() throws Exception {// GET /next on a background thread — parks in waitingContexts (no thread held).HttpRequest request = HttpRequest.newBuilder().uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))@@ -260,14 +593,49 @@ class RuntimeApiServerTest {Thread.sleep(500);assertFalse(asyncResponse.isDone(), "handler should be parked");+ // Quiesce leaves the parked poller alone — real Lambda relies on the container+ // process exiting on SIGTERM to terminate the poll, and the server can't send a+ // response that AWS doesn't document. close() then drops the underlying HTTP+ // server, terminating the parked poller's TCP connection from the server side+ // (the container-exit path in real usage).long start = System.currentTimeMillis();- server.stop();- HttpResponse<String> response = asyncResponse.get(2, TimeUnit.SECONDS);+ server.quiesce();+ server.close().get(2, TimeUnit.SECONDS);long elapsed = System.currentTimeMillis() - start;- // 204 is only valid on shutdown — the container is being terminated.- assertEquals(204, response.statusCode());- assertTrue(elapsed < 1000, "stop() should wake parked poller in <1s, took " + elapsed + "ms");+ // The client sees the socket close, surfacing as a completion exception on the+ // async future rather than a normal response. If quiesce() had (wrongly) responded+ // with a documented status code, the future would complete successfully instead.+ assertThrows(Exception.class, () -> asyncResponse.get(2, TimeUnit.SECONDS));+ assertTrue(elapsed < 3000, "close() should terminate parked poller in <3s, took " + elapsed + "ms");+ }++ @Test+ @Timeout(15)+ void quiesceLeavesSocketOpenForOrderlyShutdown() throws Exception {+ // Real teardown is quiesce() → SIGTERM container → close(). The middle step+ // relies on the runtime API socket still being bound so the runtime process+ // (mid-poll on /invocation/next) can receive SIGTERM without seeing a+ // network error first. Verify that the port is still bound after quiesce().+ server.quiesce();++ // A fresh connection can still reach the server. Handler will park it (server is+ // stopped, no work available) — we only care that the TCP handshake and HTTP+ // request-line get through, proving the listener is up.+ HttpRequest probe = HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .timeout(java.time.Duration.ofMillis(500))+ .GET()+ .build();+ CompletableFuture<HttpResponse<String>> probeFuture =+ httpClient.sendAsync(probe, HttpResponse.BodyHandlers.ofString());++ // The request should reach the server (parking there) rather than fail on connect.+ // Timing out on the client side proves the connection was accepted.+ assertThrows(Exception.class, () -> probeFuture.get(1, TimeUnit.SECONDS));++ // Now close() should complete cleanly, releasing the port.+ server.close().get(2, TimeUnit.SECONDS);}@Test