Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): SIGTERM container before closing runtime API socket (#2098)

Guinevere Friedmanb911ccd01392
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 3e1ad97f85..7eb9606a55 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -162,11 +162,23 @@ public class ContainerLifecycleManager {
* @param logStream optional log stream to close (may be null)
*/
public void stopAndRemove(String containerId, Closeable logStream) {
+ stopAndRemove(containerId, logStream, 5);
+ }
+
+ /**
+ * Stops and removes a container, closing any associated log stream.
+ *
+ * @param containerId the container ID to stop and remove
+ * @param logStream optional log stream to close (may be null)
+ * @param stopTimeoutSeconds seconds to wait for the container to stop cleanly after
+ * SIGTERM before Docker sends SIGKILL. Must be a non-negative integer.
+ */
+ public void stopAndRemove(String containerId, Closeable logStream, int stopTimeoutSeconds) {
LOG.infov("Stopping container {0}", containerId);
boolean stoppedOrMissing = false;
try {
- dockerClient.stopContainerCmd(containerId).withTimeout(5).exec();
+ dockerClient.stopContainerCmd(containerId).withTimeout(stopTimeoutSeconds).exec();
stoppedOrMissing = true;
} catch (NotFoundException e) {
LOG.debugv("Container {0} not found (already removed)", containerId);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index 94df0c6596..9cba571cec 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -445,17 +445,43 @@ public class ContainerLauncher {
LOG.infov("Stopping container {0}", handle.getContainerId());
handle.setState(ContainerState.STOPPED);
+ RuntimeApiServer server = handle.getRuntimeApiServer();
+
+ // Quiesce first: stop accepting new work, notify extensions, complete pending
+ // invocations. Existing runtime pollers on /invocation/next stay parked.
+ server.quiesce();
+
+ // Then SIGTERM the container. The runtime library's default SIGTERM handler exits
+ // the process while the runtime API socket is still up, matching real AWS Lambda's
+ // shutdown flow.
+ //
+ // stopTimeoutSeconds is the ceiling docker waits for PID 1 to exit before SIGKILL,
+ // not a grace floor: the runtime typically exits promptly, so `docker stop`
+ // returns shortly after and this value rarely matters end-to-end. Sized against
+ // AWS's documented Shutdown-phase tiers as a defensive upper bound only — 2s
+ // when at least one extension is registered (matching AWS's 2000ms
+ // external-extensions budget), else 1s. NOTE: extensions run as sibling
+ // `docker exec`s (not children of PID 1) and are killed when the container
+ // exits; the 2s ceiling does not provide a guaranteed grace window for
+ // extension shutdown work — only that we will not preempt PID 1 for at least
+ // that long if it happens to be slow.
+ int stopTimeoutSeconds = server.hasRegisteredExtensions() ? 2 : 1;
+ lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream(),
+ stopTimeoutSeconds);
+
+ // Only now close the runtime API socket. Any pollers that were still parked when
+ // the container exited get terminated by their end of the connection closing;
+ // this call releases the port for reuse.
try {
- handle.getRuntimeApiServer().stop().get(5, TimeUnit.SECONDS);
+ server.close().get(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException | TimeoutException e) {
LOG.warnv(e, "RuntimeApiServer did not close cleanly for container {0}",
handle.getContainerId());
} finally {
- runtimeApiServerFactory.release(handle.getRuntimeApiServer());
+ runtimeApiServerFactory.release(server);
}
- lifecycleManager.stopAndRemove(handle.getContainerId(), handle.getLogStream());
}
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
index 49b4aa33e4..7349a463b4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
@@ -127,6 +127,72 @@ public class RuntimeApiServer {
private boolean stopped;
private CompletableFuture<Void> closeFuture;
+ // Set by quiesce() (under lock); completes when every parked extension SHUTDOWN
+ // has flushed. close() awaits this before dropping the httpServer (see #2142).
+ // Null if quiesce() has not run.
+ private CompletableFuture<Void> extensionWritesFlushed;
+
+ /** Test-only: number of /next pollers parked in waitingContexts, for
+ * tests that want to await the parked state deterministically without sleeping. */
+ int waitingContextsSize() {
+ synchronized (lock) {
+ return waitingContexts.size();
+ }
+ }
+
+ /** Test-only: number of invocations queued waiting for a /next poller. Used by
+ * the requeue-on-write-failure test to verify the requeue actually happened. */
+ int pendingQueueSize() {
+ synchronized (lock) {
+ return pendingQueue.size();
+ }
+ }
+
+ /** Test-only: number of invocations currently in flight. Used alongside
+ * {@link #pendingQueueSize()} to verify a requeue-on-failure both offered
+ * to pendingQueue AND cleared the stale inFlight entry. */
+ int inFlightSize() {
+ return inFlight.size();
+ }
+
+ /**
+ * Test-only: called at the start of enqueue()'s deferred sendInvocation,
+ * before the guard's `stopped` re-check. A test subclass overrides this to
+ * gate the dispatch on a CountDownLatch and drive a specific interleaving
+ * with quiesce(). Empty no-op in production.
+ */
+ protected void beforeEnqueueDeferredDispatch() { }
+
+ /**
+ * Test-only: called at the start of NEXT_PATH's post-lock guard block, before
+ * the guard's `stopped` re-check. Same shape as {@link #beforeEnqueueDeferredDispatch}
+ * but for the synchronous handler-thread dispatch path.
+ */
+ protected void beforeNextPathDispatchGuard() { }
+
+ /**
+ * Test-only: called inside quiesce() immediately after the lock has been
+ * released, before any outside-lock work runs. Test subclasses override
+ * to freeze quiesce at that point and drive a dispatch through the guard
+ * against a supposedly-partially-swept state, proving atomicity holds.
+ */
+ protected void afterQuiesceStoppedFlagSet() { }
+
+ /**
+ * Test-only: called inside enqueue()'s deferred dispatch after lock release,
+ * before the send/requeue decision is acted on. Test subclasses override to
+ * mutate the parked ctx (e.g. end it) and prove the decision was committed
+ * atomically inside the lock.
+ */
+ protected void afterEnqueueDispatchLockReleased(RoutingContext waitingCtx) { }
+
+ /**
+ * Test-only: called at the top of sendInvocation so tests can observe whether
+ * a dispatch actually ran. Distinguishes "sent to a dead ctx" from "skipped,
+ * stranded" — both leave the invocation in inFlight otherwise.
+ */
+ protected void beforeSendInvocationWrite(String requestId) { }
+
// Set once an extension reports an init/exit error. Real AWS treats both as fatal to the
// execution environment, so the container must neither accept new work nor be reused.
//
@@ -183,6 +249,18 @@ public class RuntimeApiServer {
return port;
}
+ /**
+ * True if at least one extension has completed registration via
+ * {@code /extension/register}. Callers use this to pick the Shutdown-phase grace tier
+ * (see AWS's documented 0/500/2000ms limits): a container with a registered extension
+ * gets the 2s external-extensions budget, otherwise the runtime-only sub-budget.
+ */
+ public boolean hasRegisteredExtensions() {
+ synchronized (lock) {
+ return !extensions.isEmpty();
+ }
+ }
+
/**
* True once a registered extension reported an init or exit error. The execution environment
* is condemned: {@code WarmPool} must not return this container to the pool or hand it out
@@ -252,25 +330,46 @@ public class RuntimeApiServer {
// exhaustion when many warm containers poll concurrently.
router.get(NEXT_PATH).handler(ctx -> {
PendingInvocation toDispatch = null;
- boolean send204 = false;
+ boolean faultedNow = false;
synchronized (lock) {
- // `faulted` alongside `stopped`: a condemned environment must not be handed work,
- // including an invocation that was queued just before the fault was reported.
- if (stopped || faulted) {
- send204 = true;
+ if (faulted) {
+ // Faulted environment: signal the fault so the runtime exits cleanly rather
+ // than parking forever. Distinct from `stopped` — stopped is a normal
+ // teardown, faulted means something invariant-breaking happened upstream.
+ faultedNow = true;
+ } else if (stopped) {
+ // Teardown in progress. Park the poll; the container process is being
+ // SIGTERMed and will exit before the socket closes, matching how real
+ // AWS Lambda tears an environment down.
+ waitingContexts.add(ctx);
} else {
toDispatch = pendingQueue.poll();
if (toDispatch == null) {
waitingContexts.add(ctx);
+ } else {
+ // Move to inFlight under the same lock so a concurrent quiesce()
+ // sweep sees it — otherwise the invocation escapes both queues
+ // and its future is never completed.
+ inFlight.put(toDispatch.getRequestId(), toDispatch);
}
}
}
- if (send204) {
+ if (faultedNow) {
ctx.response().setStatusCode(204).end();
} else if (toDispatch != null) {
- sendInvocation(ctx, toDispatch);
+ beforeNextPathDispatchGuard();
+ // Re-check `stopped` under the lock: quiesce() may have run since our
+ // poll above, in which case it completed toDispatch's future with
+ // ContainerStopped and dispatching now would silently discard /response.
+ boolean alreadyHandled;
+ synchronized (lock) {
+ alreadyHandled = stopped;
+ }
+ if (!alreadyHandled) {
+ sendInvocation(ctx, toDispatch);
+ }
}
- // else: parked — enqueue() or stop() will dispatch this ctx later.
+ // else: parked — enqueue() or the httpServer close will dispatch/end this ctx.
});
// POST /runtime/invocation/{requestId}/response — success
@@ -479,22 +578,31 @@ public class RuntimeApiServer {
});
}
- public CompletableFuture<Void> stop() {
- CompletableFuture<Void> closed;
- List<RoutingContext> contextsToClose204;
- List<Supplier<Future<Void>>> dispatches = new ArrayList<>();
+ /**
+ * Mark the server stopped and settle all bookkeeping, without closing the underlying HTTP
+ * server. Idempotent — repeated calls are no-ops.
+ *
+ * Runtime pollers already parked on {@code /invocation/next} stay parked; the caller is
+ * expected to shut the container process down (docker stop → SIGTERM) before invoking
+ * {@link #close()}, so those pollers are terminated by the container exit rather than by
+ * a server-side response. Extensions parked on {@code /event/next} get a {@code Shutdown}
+ * event (this is documented AWS behavior for the Extensions API). Pending and in-flight
+ * invocations are completed with {@code ContainerStopped}.
+ *
+ * Call {@link #close()} afterward to release the port.
+ */
+ public void quiesce() {
+ List<Supplier<Future<Void>>> extensionDispatches = new ArrayList<>();
List<PendingInvocation> invocationsToFailContainerStopped;
+ List<PendingInvocation> inFlightToFail;
+ CompletableFuture<Void> writesFuture = new CompletableFuture<>();
synchronized (lock) {
- if (closeFuture != null) {
- return closeFuture;
+ if (stopped) {
+ return;
}
stopped = true;
- closed = new CompletableFuture<>();
- closeFuture = closed;
-
- contextsToClose204 = new ArrayList<>(waitingContexts);
- waitingContexts.clear();
+ extensionWritesFlushed = writesFuture;
ExtensionEvent shutdownEvent = ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN");
for (RegisteredExtension ext : extensions.values()) {
@@ -503,7 +611,7 @@ public class RuntimeApiServer {
}
RoutingContext waitingCtx = ext.takeWaitingContext();
if (waitingCtx != null) {
- dispatches.add(() -> {
+ extensionDispatches.add(() -> {
if (waitingCtx.response().ended()) {
return Future.succeededFuture();
}
@@ -516,6 +624,75 @@ public class RuntimeApiServer {
invocationsToFailContainerStopped = new ArrayList<>(pendingQueue);
pendingQueue.clear();
+ // Snapshot and clear inFlight inside the lock, atomic with stopped=true.
+ // The dispatch guards in NEXT_PATH and enqueue()'s deferred callback rely on
+ // this: once quiesce releases the lock, stopped=true implies inFlight is
+ // empty, so a guard reading `stopped` cannot deliver an invocation whose
+ // future quiesce is about to complete with ContainerStopped.
+ inFlightToFail = new ArrayList<>(inFlight.values());
+ inFlight.clear();
+ }
+ afterQuiesceStoppedFlagSet();
+
+ // Extension SHUTDOWN writes go through the event loop; chain on end() rather
+ // than the scheduled handler's return, because response.end() is async and the
+ // bytes are not on the wire until its future completes. close() awaits
+ // extensionWritesFlushed before dropping the httpServer (see #2142).
+ //
+ // Runtime pollers stay parked — no 204. They are terminated either by the
+ // container process exiting on SIGTERM (the ContainerLauncher path, which
+ // interleaves docker stop between quiesce and close) or by the httpServer
+ // close itself (the stop() wrapper path); either matches real AWS Lambda's
+ // teardown better than an undocumented 204 on /invocation/next would.
+ if (extensionDispatches.isEmpty()) {
+ writesFuture.complete(null);
+ } else {
+ AtomicInteger remainingWrites = new AtomicInteger(extensionDispatches.size());
+ for (Supplier<Future<Void>> dispatch : extensionDispatches) {
+ vertx.runOnContext(v -> dispatch.get().onComplete(ar -> {
+ if (remainingWrites.decrementAndGet() == 0) {
+ writesFuture.complete(null);
+ }
+ }));
+ }
+ }
+
+ // Fail queued invocations that were never consumed by /next.
+ for (PendingInvocation pending : invocationsToFailContainerStopped) {
+ pending.getResultFuture().complete(
+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));
+ }
+
+ // Complete any in-flight invocations with error.
+ for (PendingInvocation inv : inFlightToFail) {
+ inv.getResultFuture().complete(
+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, inv.getRequestId()));
+ }
+ }
+
+ /**
+ * Close the underlying HTTP server, releasing the bound port. Idempotent — the returned
+ * future resolves once the server has actually closed (first call) or immediately with
+ * the same future (subsequent calls). Any parked runtime pollers are terminated by the
+ * underlying channel closing.
+ *
+ * {@link #quiesce()} should be called first so that any registered extensions have the
+ * chance to receive a Shutdown event before their sockets die.
+ */
+ public CompletableFuture<Void> close() {
+ CompletableFuture<Void> closed;
+ CompletableFuture<Void> awaitWrites;
+ synchronized (lock) {
+ if (closeFuture != null) {
+ return closeFuture;
+ }
+ closed = new CompletableFuture<>();
+ closeFuture = closed;
+ // Await any extension SHUTDOWN writes quiesce() scheduled before closing
+ // the httpServer, so their responses don't race the socket teardown.
+ awaitWrites = extensionWritesFlushed != null
+ ? extensionWritesFlushed
+ : CompletableFuture.completedFuture(null);
}
Runnable closeServer = () -> {
@@ -533,54 +710,21 @@ public class RuntimeApiServer {
closed.complete(null);
}
};
-
- // Wake any parked /next pollers with 204 (container shutting down — runtime will exit) and
- // dispatch SHUTDOWN to every extension already parked on /event/next, then close the HTTP
- // server. Each write is scheduled on the event loop via vertx.runOnContext(...), and the
- // server is closed only once every write's response has actually been flushed — tracked by
- // remainingWrites, decremented from each end() future's completion rather than when the
- // scheduled handler returns (response.end() is asynchronous, so the bytes are not on the
- // wire yet when the handler returns). So a parked extension's SHUTDOWN — or a poller's 204
- // — is fully written before its connection is torn down, instead of racing
- // httpServer.close() and orphaning the extension with an EOF. See issue #2142.
- int parkedWrites = contextsToClose204.size() + dispatches.size();
- if (parkedWrites == 0) {
- closeServer.run();
- } else {
- AtomicInteger remainingWrites = new AtomicInteger(parkedWrites);
- Runnable afterWrite = () -> {
- if (remainingWrites.decrementAndGet() == 0) {
- closeServer.run();
- }
- };
- for (RoutingContext ctx : contextsToClose204) {
- vertx.runOnContext(v -> {
- Future<Void> written = ctx.response().ended()
- ? Future.succeededFuture()
- : ctx.response().setStatusCode(204).end();
- written.onComplete(ar -> afterWrite.run());
- });
- }
- for (Supplier<Future<Void>> dispatch : dispatches) {
- vertx.runOnContext(v -> dispatch.get().onComplete(ar -> afterWrite.run()));
- }
- }
-
- // Fail queued invocations that were never consumed by /next.
- for (PendingInvocation pending : invocationsToFailContainerStopped) {
- pending.getResultFuture().complete(
- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));
- }
-
- // Complete any in-flight invocations with error.
- inFlight.values().forEach(inv ->
- inv.getResultFuture().complete(
- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, inv.getRequestId())));
- inFlight.clear();
-
+ awaitWrites.whenComplete((v, err) -> closeServer.run());
return closed;
}
+ /**
+ * Convenience wrapper: {@link #quiesce()} then {@link #close()} back-to-back, matching the
+ * pre-split single-step teardown. Callers that need to SIGTERM the container between the
+ * quiesce and close phases (which is the AWS-faithful teardown) should call the two
+ * methods directly rather than this wrapper.
+ */
+ public CompletableFuture<Void> stop() {
+ quiesce();
+ return close();
+ }
+
public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {
boolean rejected;
List<Runnable> dispatches = new ArrayList<>();
@@ -621,6 +765,11 @@ public class RuntimeApiServer {
waitingCtxForInvocation = waitingContexts.poll();
if (waitingCtxForInvocation == null) {
pendingQueue.offer(invocation);
+ } else {
+ // Move to inFlight under the same lock so a concurrent quiesce()
+ // sweep sees it — otherwise the invocation escapes both queues and
+ // the caller hangs until the function deadline.
+ inFlight.put(invocation.getRequestId(), invocation);
}
}
}
@@ -638,14 +787,39 @@ public class RuntimeApiServer {
if (waitingCtxForInvocation != null) {
final RoutingContext waitingCtx = waitingCtxForInvocation;
vertx.runOnContext(v -> {
- if (!waitingCtx.response().ended()) {
- sendInvocation(waitingCtx, invocation);
- } else {
- // Connection closed between park and dispatch — re-queue.
- synchronized (lock) {
+ beforeEnqueueDeferredDispatch();
+ // Two things could have changed between enqueue()'s lock release and
+ // this deferred dispatch reaching the event loop:
+ // 1. quiesce() may have run — atomic with clearing inFlight and
+ // completing the caller's future with ContainerStopped. Skip.
+ // 2. The parked client may have disconnected — the ctx is ended and
+ // writing to it would fail. Requeue for the next /next poller
+ // and drop this stale inFlight entry (the next poller will re-put
+ // the invocation into inFlight under its own lock).
+ //
+ // Read waitingCtx.response().ended() ONCE, under the lock, so the
+ // dispatch/requeue decision is committed atomically. Reading it a
+ // second time outside the lock (as the code did previously) let a
+ // disconnect between the two reads produce a stranded inFlight entry:
+ // the inside-lock read saw !ended → no requeue; the outside-lock read
+ // saw ended → no send; neither branch fired.
+ boolean shouldDispatch;
+ synchronized (lock) {
+ if (stopped) {
+ return;
+ }
+ if (waitingCtx.response().ended()) {
pendingQueue.offer(invocation);
+ inFlight.remove(invocation.getRequestId());
+ shouldDispatch = false;
+ } else {
+ shouldDispatch = true;
}
}
+ afterEnqueueDispatchLockReleased(waitingCtx);
+ if (shouldDispatch) {
+ sendInvocation(waitingCtx, invocation);
+ }
});
}
return invocation.getResultFuture();
@@ -677,19 +851,40 @@ public class RuntimeApiServer {
}
private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {
- inFlight.put(invocation.getRequestId(), invocation);
-
+ beforeSendInvocationWrite(invocation.getRequestId());
+ // Invariant: callers put the invocation in inFlight under the lock before
+ // dispatching, so no inFlight.put here.
byte[] payload = invocation.getPayload();
String body = (payload != null && payload.length > 0)
? new String(payload)
: "{}";
- ctx.response()
- .setStatusCode(200)
- .putHeader("Content-Type", "application/json")
- .putHeader("Lambda-Runtime-Aws-Request-Id", invocation.getRequestId())
- .putHeader("Lambda-Runtime-Invoked-Function-Arn", invocation.getFunctionArn())
- .putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs()))
- .end(body);
+ // Vert.x reports a client-side disconnect two ways: setStatusCode/putHeader
+ // throw synchronously if the response is already ended, .end() returns a
+ // failed Future if the socket dies mid-flush. Funnel both into one handler.
+ Future<Void> write;
+ try {
+ write = ctx.response()
+ .setStatusCode(200)
+ .putHeader("Content-Type", "application/json")
+ .putHeader("Lambda-Runtime-Aws-Request-Id", invocation.getRequestId())
+ .putHeader("Lambda-Runtime-Invoked-Function-Arn", invocation.getFunctionArn())
+ .putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs()))
+ .end(body);
+ } catch (IllegalStateException alreadyEnded) {
+ write = Future.failedFuture(alreadyEnded);
+ }
+ write.onFailure(err -> {
+ // Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.
+ // Skip if quiesce() beat us — it already swept inFlight and settled the
+ // future with ContainerStopped.
+ synchronized (lock) {
+ if (stopped) {
+ return;
+ }
+ pendingQueue.offer(invocation);
+ inFlight.remove(invocation.getRequestId());
+ }
+ });
}
private Future<Void> sendExtensionEvent(RoutingContext ctx, ExtensionEvent event) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
index 8cb210c951..01e76002cc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
@@ -20,6 +20,7 @@ import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -67,6 +68,22 @@ class RuntimeApiServerTest {
vertx.close().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
}
+ /**
+ * Polls until at least {@code count} runtime pollers have parked in the server's
+ * {@code waitingContexts}. Replaces Thread.sleep-based waits so tests don't guess
+ * how long the client-side TCP connect + server-side park sequence takes.
+ */
+ private void awaitWaitingContexts(int count) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 5000;
+ while (System.currentTimeMillis() < deadline) {
+ if (server.waitingContextsSize() >= count) return;
+ Thread.sleep(10);
+ }
+ throw new AssertionError(
+ "expected at least " + count + " parked /next poller(s); got "
+ + server.waitingContextsSize());
+ }
+
@Test
@Timeout(15)
void nextEndpoint_blocksUntilInvocationArrives() throws Exception {
@@ -245,9 +262,325 @@ class RuntimeApiServerTest {
assertTrue(payload.contains("ContainerStopped"));
}
+ /**
+ * Replaces the default {@code server} with a subclass whose test-only overrides
+ * gate the four race points on the caller-supplied Runnables. Used only by the
+ * race tests below; other tests keep the plain server. Rebinds the port because
+ * start() binds on construction.
+ */
+ private void installGatedServer(Runnable beforeEnqueueDispatch,
+ Runnable beforeNextPathGuard,
+ Runnable afterQuiesceStoppedFlag) throws Exception {
+ server.stop().get(5, TimeUnit.SECONDS);
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port) {
+ @Override protected void beforeEnqueueDeferredDispatch() { beforeEnqueueDispatch.run(); }
+ @Override protected void beforeNextPathDispatchGuard() { beforeNextPathGuard.run(); }
+ @Override protected void afterQuiesceStoppedFlagSet() { afterQuiesceStoppedFlag.run(); }
+ };
+ server.start().get(5, TimeUnit.SECONDS);
+ }
+
+ /**
+ * Variant of {@link #installGatedServer} for the ctx-ends-during-dispatch race:
+ * inject state after enqueue()'s dispatch lock is released, observe whether
+ * sendInvocation subsequently ran.
+ */
+ private void installGatedServerWithDispatchObservation(
+ java.util.function.Consumer<io.vertx.ext.web.RoutingContext> afterEnqueueLockReleased,
+ java.util.function.Consumer<String> onSendInvocation) throws Exception {
+ server.stop().get(5, TimeUnit.SECONDS);
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port) {
+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {
+ afterEnqueueLockReleased.accept(waitingCtx);
+ }
+ @Override protected void beforeSendInvocationWrite(String requestId) {
+ onSendInvocation.accept(requestId);
+ }
+ };
+ server.start().get(5, TimeUnit.SECONDS);
+ }
+
@Test
@Timeout(15)
- void stopWakesParkedPollerImmediately() throws Exception {
+ void enqueueDeferredDispatch_racedByQuiesce_doesNotDeliverAfterSweep() throws Exception {
+ // The dispatch guard is only sound if quiesce()'s inFlight sweep is atomic
+ // with stopped=true under the lock. Prove that by driving the exact
+ // interleaving the pre-hampsterx-fix code exposed. Sequence:
+ //
+ // 1. Park a /next poller server-side; enqueue an invocation → puts inv into
+ // inFlight under the lock, schedules a deferred sendInvocation on the
+ // event loop.
+ // 2. The deferred dispatch fires but blocks on `holdDispatch` before it
+ // can reach the guard's stopped-recheck.
+ // 3. Run quiesce() on a background thread. Sets stopped=true, sweeps
+ // inFlight (atomically, under fix) or defers it (pre-fix), releases
+ // lock. Blocks at afterQuiesceStoppedFlagSet.
+ // 4. Release `holdDispatch`. Guard reads `stopped`: under the fix, sees
+ // atomically-cleared state and skips. Under the pre-fix code, sweep
+ // hasn't run outside the lock yet — old guard `stopped && inFlight
+ // .get(id)==null` would be false and dispatch would deliver.
+ // 5. Release quiesce.
+ //
+ // Assertion: parked client MUST NOT have received a 200 with our request-id.
+ CountDownLatch dispatchEntered = new CountDownLatch(1);
+ CountDownLatch holdDispatch = new CountDownLatch(1);
+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);
+ CountDownLatch releaseQuiesce = new CountDownLatch(1);
+ installGatedServer(
+ () -> {
+ dispatchEntered.countDown();
+ try { holdDispatch.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ },
+ () -> { /* NEXT_PATH not exercised here */ },
+ () -> {
+ quiesceReachedHook.countDown();
+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+
+ HttpRequest parkedRequest = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build();
+ CompletableFuture<HttpResponse<String>> parkedResponse =
+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-latch-enqueue", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+ assertTrue(dispatchEntered.await(5, TimeUnit.SECONDS),
+ "deferred dispatch should have entered the pre-guard hook");
+
+ // Kick off quiesce on a background thread; it will freeze at the post-lock hook.
+ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(() -> server.quiesce());
+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),
+ "quiesce should have released its lock and reached the post-lock hook");
+
+ // NOW release the dispatch while quiesce is frozen post-lock — the exact window
+ // the pre-fix code was vulnerable in. Guard's `stopped=true` under fix must
+ // atomically imply inFlight empty.
+ holdDispatch.countDown();
+ // Give the guard time to run (blocking on releaseQuiesce first).
+ Thread.sleep(200);
+ releaseQuiesce.countDown();
+ quiesceDone.get(5, TimeUnit.SECONDS);
+
+ try {
+ HttpResponse<String> response = parkedResponse.get(2, TimeUnit.SECONDS);
+ assertFalse(response.statusCode() == 200
+ && "req-latch-enqueue".equals(response.headers()
+ .firstValue("Lambda-Runtime-Aws-Request-Id").orElse("")),
+ "guard failed: parked poller received the invocation after quiesce; "
+ + "would cause silent discard of /response. Got "
+ + response.statusCode());
+ } catch (Exception expected) {
+ // socket close from tearDown — expected shape.
+ }
+ InvokeResult result = invocation.getResultFuture().get(2, TimeUnit.SECONDS);
+ assertEquals("Unhandled", result.getFunctionError());
+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));
+ }
+
+ @Test
+ @Timeout(15)
+ void nextPathDispatch_racedByQuiesce_doesNotDeliverAfterSweep() throws Exception {
+ // Symmetric to the enqueue race but on NEXT_PATH: an invocation sits in
+ // pendingQueue. The handler polls it out under the lock, moves to inFlight,
+ // releases lock, then reaches the guard. We hold the guard, run quiesce on
+ // a background thread to freeze it post-lock, release the guard.
+ CountDownLatch dispatchEntered = new CountDownLatch(1);
+ CountDownLatch holdDispatch = new CountDownLatch(1);
+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);
+ CountDownLatch releaseQuiesce = new CountDownLatch(1);
+ installGatedServer(
+ () -> { /* enqueue path not exercised here */ },
+ () -> {
+ dispatchEntered.countDown();
+ try { holdDispatch.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ },
+ () -> {
+ quiesceReachedHook.countDown();
+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-latch-nextpath", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ HttpRequest nextRequest = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build();
+ CompletableFuture<HttpResponse<String>> nextResponse =
+ httpClient.sendAsync(nextRequest, HttpResponse.BodyHandlers.ofString());
+ assertTrue(dispatchEntered.await(5, TimeUnit.SECONDS),
+ "NEXT_PATH handler should have entered the pre-guard hook");
+
+ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(() -> server.quiesce());
+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),
+ "quiesce should have released its lock and reached the post-lock hook");
+
+ holdDispatch.countDown();
+ Thread.sleep(200);
+ releaseQuiesce.countDown();
+ quiesceDone.get(5, TimeUnit.SECONDS);
+
+ try {
+ HttpResponse<String> response = nextResponse.get(2, TimeUnit.SECONDS);
+ assertFalse(response.statusCode() == 200
+ && "req-latch-nextpath".equals(response.headers()
+ .firstValue("Lambda-Runtime-Aws-Request-Id").orElse("")),
+ "guard failed: /next received the invocation after quiesce; "
+ + "would cause silent discard of /response. Got "
+ + response.statusCode());
+ } catch (Exception expected) {
+ // socket close from tearDown — expected shape.
+ }
+ InvokeResult result = invocation.getResultFuture().get(2, TimeUnit.SECONDS);
+ assertEquals("Unhandled", result.getFunctionError());
+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));
+ }
+
+ @Test
+ @Timeout(15)
+ void enqueueDeferredDispatch_ctxEndsAfterLockRelease_dispatchesOrRequeuesAtomically() throws Exception {
+ // Pre-fix, enqueue()'s deferred callback read waitingCtx.response().ended()
+ // twice — once inside the lock deciding requeue, once outside deciding send.
+ // A disconnect between the two reads made neither branch fire and stranded
+ // the invocation in inFlight until the function deadline. Force that
+ // interleaving by ending the ctx from afterEnqueueDispatchLockReleased.
+ java.util.concurrent.atomic.AtomicInteger sendInvocationCalls =
+ new java.util.concurrent.atomic.AtomicInteger();
+ installGatedServerWithDispatchObservation(
+ waitingCtx -> {
+ if (!waitingCtx.response().ended()) {
+ waitingCtx.response().setStatusCode(500).end();
+ }
+ },
+ requestId -> sendInvocationCalls.incrementAndGet());
+
+ HttpRequest parkedRequest = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build();
+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-ctx-ends", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ // Wait for the runOnContext-scheduled dispatch to have fired.
+ Thread.sleep(500);
+
+ assertEquals(1, sendInvocationCalls.get(),
+ "sendInvocation must run exactly once — a zero count means the "
+ + "invocation was stranded (the pre-fix stranded-inFlight bug).");
+ }
+
+ @Test
+ @Timeout(15)
+ void sendInvocation_writeFails_requeuesAndClearsInFlight() throws Exception {
+ // Post-atomic-decision race: disconnect between dispatch commitment and
+ // .end() flush strands the invocation in inFlight. Force it by ending the
+ // parked ctx from beforeSendInvocationWrite; assert onFailure requeues +
+ // clears inFlight.
+ java.util.concurrent.atomic.AtomicReference<io.vertx.ext.web.RoutingContext> parkedCtx =
+ new java.util.concurrent.atomic.AtomicReference<>();
+ server.stop().get(5, TimeUnit.SECONDS);
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port) {
+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {
+ parkedCtx.set(waitingCtx);
+ }
+ @Override protected void beforeSendInvocationWrite(String requestId) {
+ io.vertx.ext.web.RoutingContext ctx = parkedCtx.get();
+ if (ctx != null && !ctx.response().ended()) {
+ ctx.response().setStatusCode(500).end();
+ }
+ }
+ };
+ server.start().get(5, TimeUnit.SECONDS);
+
+ HttpRequest parkedRequest = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build();
+ httpClient.sendAsync(parkedRequest, HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-write-fails", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ // Wait for the runOnContext-scheduled dispatch (and its failed write) to fire.
+ Thread.sleep(500);
+
+ assertEquals(1, server.pendingQueueSize(),
+ "invocation must be requeued after write failure — zero means stranded.");
+ assertEquals(0, server.inFlightSize(),
+ "inFlight must be cleared alongside the requeue.");
+ }
+
+ @Test
+ @Timeout(15)
+ void quiesceAtomicallyClearsInFlightWithStopped() throws Exception {
+ // The dispatch guards in enqueue()'s deferred callback and NEXT_PATH read
+ // `stopped` under the server lock and act on it. Their correctness depends on
+ // an invariant: once quiesce() releases its lock, `stopped=true` implies
+ // inFlight is empty. If quiesce() cleared inFlight *outside* the lock, a
+ // dispatch could observe stopped=true while an inFlight entry it's about to
+ // deliver was still present — the guard would let the delivery through,
+ // quiesce would then complete the future with ContainerStopped, and the
+ // runtime's /response would be silently discarded.
+ //
+ // Put entries in inFlight via full enqueue → /next round trips (matches the
+ // real code path), then call quiesce and assert all their futures completed.
+ List<PendingInvocation> invocations = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ PendingInvocation inv = new PendingInvocation(
+ "req-atomic-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(inv);
+ invocations.add(inv);
+ HttpResponse<String> resp = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port
+ + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, resp.statusCode());
+ }
+
+ server.quiesce();
+
+ // quiesce() sets stopped=true AND clears inFlight in the same synchronized
+ // block, so both observations are atomic. Every invocation must be completed.
+ for (PendingInvocation inv : invocations) {
+ InvokeResult result = inv.getResultFuture().get(2, TimeUnit.SECONDS);
+ assertEquals("Unhandled", result.getFunctionError(),
+ "invocation " + inv.getRequestId() + " must be completed by quiesce");
+ assertTrue(new String(result.getPayload()).contains("ContainerStopped"));
+ }
+ }
+
+ @Test
+ @Timeout(15)
+ void closeTerminatesParkedPollerWithoutResponse() throws Exception {
// GET /next on a background thread — parks in waitingContexts (no thread held).
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
@@ -260,14 +593,49 @@ class RuntimeApiServerTest {
Thread.sleep(500);
assertFalse(asyncResponse.isDone(), "handler should be parked");
+ // Quiesce leaves the parked poller alone — real Lambda relies on the container
+ // process exiting on SIGTERM to terminate the poll, and the server can't send a
+ // response that AWS doesn't document. close() then drops the underlying HTTP
+ // server, terminating the parked poller's TCP connection from the server side
+ // (the container-exit path in real usage).
long start = System.currentTimeMillis();
- server.stop();
- HttpResponse<String> response = asyncResponse.get(2, TimeUnit.SECONDS);
+ server.quiesce();
+ server.close().get(2, TimeUnit.SECONDS);
long elapsed = System.currentTimeMillis() - start;
- // 204 is only valid on shutdown — the container is being terminated.
- assertEquals(204, response.statusCode());
- assertTrue(elapsed < 1000, "stop() should wake parked poller in <1s, took " + elapsed + "ms");
+ // The client sees the socket close, surfacing as a completion exception on the
+ // async future rather than a normal response. If quiesce() had (wrongly) responded
+ // with a documented status code, the future would complete successfully instead.
+ assertThrows(Exception.class, () -> asyncResponse.get(2, TimeUnit.SECONDS));
+ assertTrue(elapsed < 3000, "close() should terminate parked poller in <3s, took " + elapsed + "ms");
+ }
+
+ @Test
+ @Timeout(15)
+ void quiesceLeavesSocketOpenForOrderlyShutdown() throws Exception {
+ // Real teardown is quiesce() → SIGTERM container → close(). The middle step
+ // relies on the runtime API socket still being bound so the runtime process
+ // (mid-poll on /invocation/next) can receive SIGTERM without seeing a
+ // network error first. Verify that the port is still bound after quiesce().
+ server.quiesce();
+
+ // A fresh connection can still reach the server. Handler will park it (server is
+ // stopped, no work available) — we only care that the TCP handshake and HTTP
+ // request-line get through, proving the listener is up.
+ HttpRequest probe = HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .timeout(java.time.Duration.ofMillis(500))
+ .GET()
+ .build();
+ CompletableFuture<HttpResponse<String>> probeFuture =
+ httpClient.sendAsync(probe, HttpResponse.BodyHandlers.ofString());
+
+ // The request should reach the server (parking there) rather than fail on connect.
+ // Timing out on the client side proves the connection was accepted.
+ assertThrows(Exception.class, () -> probeFuture.get(1, TimeUnit.SECONDS));
+
+ // Now close() should complete cleanly, releasing the port.
+ server.close().get(2, TimeUnit.SECONDS);
}
@Test