Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): deliver extension INVOKE when the runtime receives the invocation (#3070)

Ammar43e2a482636b
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
index 7349a463b4..da4f2d15d1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
@@ -725,9 +725,53 @@ public class RuntimeApiServer {
return close();
}
+ /**
+ * Fan the INVOKE event out to every subscribed extension. Called from
+ * {@link #sendInvocation(RoutingContext, PendingInvocation)}, i.e. at the moment the runtime
+ * actually receives the invocation, not at enqueue() time. Nothing in the launch path waits for
+ * an internal extension to register, so an extension registering from inside the runtime process
+ * may not be in {@code extensions} yet when enqueue() runs; dispatching there dropped the event
+ * entirely (see #2573).
+ *
+ * <p>{@code stopped} is read under the lock for the same reason sendInvocation()'s failure path
+ * reads it: quiesce() may already have swept the extensions and queued SHUTDOWN, and an INVOKE
+ * offered afterwards would be drained ahead of that SHUTDOWN by /extension/event/next.
+ */
+ private void notifyExtensionsOfInvoke(PendingInvocation invocation) {
+ List<Runnable> dispatches = new ArrayList<>();
+ synchronized (lock) {
+ if (stopped || extensions.isEmpty()) {
+ return;
+ }
+ ExtensionEvent event = ExtensionEvent.invoke(
+ invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());
+ for (RegisteredExtension ext : extensions.values()) {
+ if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {
+ continue;
+ }
+ RoutingContext waitingCtx = ext.takeWaitingContext();
+ if (waitingCtx != null) {
+ dispatches.add(() -> {
+ if (!waitingCtx.response().ended()) {
+ sendExtensionEvent(waitingCtx, event);
+ } else {
+ synchronized (lock) {
+ ext.getPendingEvents().offer(event);
+ }
+ }
+ });
+ } else {
+ ext.getPendingEvents().offer(event);
+ }
+ }
+ }
+ for (Runnable dispatch : dispatches) {
+ vertx.runOnContext(v -> dispatch.run());
+ }
+ }
+
public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {
boolean rejected;
- List<Runnable> dispatches = new ArrayList<>();
RoutingContext waitingCtxForInvocation = null;
synchronized (lock) {
@@ -738,30 +782,6 @@ public class RuntimeApiServer {
// stays a single atomic step.
rejected = stopped || faulted;
if (!rejected) {
- if (!extensions.isEmpty()) {
- ExtensionEvent event = ExtensionEvent.invoke(
- invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());
- for (RegisteredExtension ext : extensions.values()) {
- if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {
- continue;
- }
- RoutingContext waitingCtx = ext.takeWaitingContext();
- if (waitingCtx != null) {
- dispatches.add(() -> {
- if (!waitingCtx.response().ended()) {
- sendExtensionEvent(waitingCtx, event);
- } else {
- synchronized (lock) {
- ext.getPendingEvents().offer(event);
- }
- }
- });
- } else {
- ext.getPendingEvents().offer(event);
- }
- }
- }
-
waitingCtxForInvocation = waitingContexts.poll();
if (waitingCtxForInvocation == null) {
pendingQueue.offer(invocation);
@@ -780,10 +800,6 @@ public class RuntimeApiServer {
return invocation.getResultFuture();
}
- for (Runnable dispatch : dispatches) {
- vertx.runOnContext(v -> dispatch.run());
- }
-
if (waitingCtxForInvocation != null) {
final RoutingContext waitingCtx = waitingCtxForInvocation;
vertx.runOnContext(v -> {
@@ -873,6 +889,9 @@ public class RuntimeApiServer {
} catch (IllegalStateException alreadyEnded) {
write = Future.failedFuture(alreadyEnded);
}
+ // Gated on the successful write: onFailure below requeues the invocation, and a fan-out
+ // before the write would emit a second INVOKE for the same requestId on redelivery.
+ write.onSuccess(v -> notifyExtensionsOfInvoke(invocation));
write.onFailure(err -> {
// Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.
// Skip if quiesce() beat us — it already swept inFlight and settled the
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
index 01e76002cc..c9fdbb8d4b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
@@ -870,6 +870,18 @@ class RuntimeApiServerTest {
assertEquals(403, response.statusCode());
}
+ /**
+ * Regression pin for #2573: the extension INVOKE fan-out fires when the runtime actually
+ * receives the invocation via /next, not at enqueue() time. Before the fix, enqueue() built
+ * and dispatched the event synchronously to whatever was in {@code extensions} at that exact
+ * moment, regardless of whether the runtime had polled yet; an internal extension (no
+ * {@code /opt/extensions} footprint, so nothing ever waits for its registration) commonly
+ * registers concurrently with or after that call and misses the event entirely, with nothing
+ * later reconsidering it. The distinguishing assertion is the one right after enqueue():
+ * on unfixed code the event is already delivered by that point (enqueue() dispatched it
+ * inline), so the test fails there; on fixed code nothing is delivered until /next is
+ * actually polled below.
+ */
@Test
@Timeout(15)
void extensionEventNext_receivesInvokeEventWhenRuntimeInvocationEnqueued() throws Exception {
@@ -890,6 +902,20 @@ class RuntimeApiServerTest {
new CompletableFuture<>());
server.enqueue(invocation);
+ // The #2573 regression check: enqueue() alone must not deliver the INVOKE. Pre-fix,
+ // this is exactly where the event arrived (dispatched inline inside enqueue()).
+ Thread.sleep(300);
+ assertFalse(asyncNext.isDone(),
+ "enqueue() alone must not deliver INVOKE; delivery must wait for the runtime's own /next poll");
+
+ // The INVOKE fan-out now happens at sendInvocation() time, so nothing is delivered until
+ // the runtime itself polls for the next invocation.
+ HttpResponse<String> nextResponse = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, nextResponse.statusCode());
+
HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);
assertEquals(200, response.statusCode());
assertTrue(response.headers().firstValue("Lambda-Extension-Event-Identifier").isPresent());
@@ -916,11 +942,284 @@ class RuntimeApiServerTest {
"arn:aws:lambda:us-east-1:000000000000:function:test",
new CompletableFuture<>()));
+ // Drive the invocation through to the runtime, same as a real dispatch would, so this
+ // exercises the actual fan-out point rather than a poller that never arrives.
+ httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
Thread.sleep(500);
assertFalse(asyncNext.isDone(),
"extension not subscribed to INVOKE must not be woken by an invocation");
}
+ /**
+ * Case 3 of the #2573 fix: an extension that has registered but not yet issued its first
+ * /event/next when the invocation dispatches must still receive it. notifyExtensionsOfInvoke()
+ * offers the event into the extension's pendingEvents queue rather than dropping it when
+ * there is no parked context to write to directly.
+ */
+ @Test
+ @Timeout(15)
+ void notifyExtensionsOfInvoke_extensionRegisteredButNotYetPolling_queuesEventForLaterPoll()
+ throws Exception {
+ String extensionId = registerExtension("lambda-adapter", "INVOKE");
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-not-yet-polling", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, next.statusCode());
+
+ // The extension's first poll arrives after the dispatch already happened, so it must find
+ // the event waiting rather than parking forever.
+ HttpResponse<String> eventResponse = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, eventResponse.statusCode());
+ JsonObject body = new JsonObject(eventResponse.body());
+ assertEquals("INVOKE", body.getString("eventType"));
+ assertEquals("req-not-yet-polling", body.getString("requestId"));
+ }
+
+ /**
+ * Case 4 of the #2573 fix: the common warm-container path, where the runtime is already
+ * parked on /next and the extension already parked on /event/next when enqueue() runs.
+ * Exercises the deferred-dispatch branch (enqueue()'s vertx.runOnContext callback) rather
+ * than the synchronous NEXT_PATH handler, and confirms relocating the fan-out to
+ * sendInvocation() did not regress this case.
+ */
+ @Test
+ @Timeout(15)
+ void notifyExtensionsOfInvoke_warmPath_runtimeAndExtensionBothParked_deliversBoth()
+ throws Exception {
+ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ String extensionId = registerExtension("lambda-adapter", "INVOKE");
+ CompletableFuture<HttpResponse<String>> asyncExtensionNext = pollExtensionEventNext(extensionId);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-warm-path", "{\"warm\":true}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ HttpResponse<String> runtimeResponse = asyncNext.get(2, TimeUnit.SECONDS);
+ assertEquals(200, runtimeResponse.statusCode());
+ assertEquals("req-warm-path",
+ runtimeResponse.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));
+
+ HttpResponse<String> extensionResponse = asyncExtensionNext.get(2, TimeUnit.SECONDS);
+ assertEquals(200, extensionResponse.statusCode());
+ JsonObject body = new JsonObject(extensionResponse.body());
+ assertEquals("INVOKE", body.getString("eventType"));
+ assertEquals("req-warm-path", body.getString("requestId"));
+ }
+
+ /**
+ * Case 2 of the #2573 fix, and the maintainer's stated requirement: sendInvocation()'s
+ * onFailure requeues the invocation for redelivery to a second /next poller. Since
+ * notifyExtensionsOfInvoke() is gated on the write's onSuccess, the failed first attempt must
+ * not have fired it. Only the second, successful write should, so the extension sees exactly
+ * one INVOKE for the requestId, never two. Forces the first write to fail the same way
+ * {@link #sendInvocation_writeFails_requeuesAndClearsInFlight} does, then counts
+ * beforeSendInvocationWrite calls to prove the redelivery actually happened rather than the
+ * assertions passing by coincidence.
+ */
+ @Test
+ @Timeout(15)
+ void notifyExtensionsOfInvoke_requeuedOnWriteFailure_deliversInvokeExactlyOnce() throws Exception {
+ java.util.concurrent.atomic.AtomicInteger sendInvocationCalls =
+ new java.util.concurrent.atomic.AtomicInteger();
+ java.util.concurrent.atomic.AtomicReference<io.vertx.ext.web.RoutingContext> parkedCtx =
+ new java.util.concurrent.atomic.AtomicReference<>();
+ server.stop().get(5, TimeUnit.SECONDS);
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port) {
+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {
+ parkedCtx.set(waitingCtx);
+ }
+ @Override protected void beforeSendInvocationWrite(String requestId) {
+ if (sendInvocationCalls.incrementAndGet() == 1) {
+ // Simulate the client disconnecting between dispatch commitment and the
+ // write landing, forcing sendInvocation()'s write to fail.
+ io.vertx.ext.web.RoutingContext ctx = parkedCtx.get();
+ if (ctx != null && !ctx.response().ended()) {
+ ctx.response().setStatusCode(500).end();
+ }
+ }
+ }
+ };
+ server.start().get(5, TimeUnit.SECONDS);
+
+ String extensionId = registerExtension("lambda-adapter", "INVOKE");
+ CompletableFuture<HttpResponse<String>> asyncExtensionNext = pollExtensionEventNext(extensionId);
+
+ httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-no-duplicate", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ // Wait for the forced-failed dispatch to requeue.
+ long deadline = System.currentTimeMillis() + 5000;
+ while (System.currentTimeMillis() < deadline && server.pendingQueueSize() == 0) {
+ Thread.sleep(10);
+ }
+ assertEquals(1, server.pendingQueueSize(),
+ "invocation must be requeued after the forced write failure");
+
+ // A second poller picks up the requeued invocation; this write succeeds.
+ HttpResponse<String> secondNext = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, secondNext.statusCode());
+
+ assertEquals(2, sendInvocationCalls.get(),
+ "sendInvocation must have been attempted twice: the failed original, and the redelivery");
+
+ HttpResponse<String> extensionResponse = asyncExtensionNext.get(2, TimeUnit.SECONDS);
+ assertEquals(200, extensionResponse.statusCode());
+ assertEquals("req-no-duplicate", new JsonObject(extensionResponse.body()).getString("requestId"));
+
+ // No second INVOKE should be waiting behind the one just delivered.
+ CompletableFuture<HttpResponse<String>> secondExtensionPoll = httpClient.sendAsync(
+ HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ Thread.sleep(300);
+ assertFalse(secondExtensionPoll.isDone(),
+ "no duplicate INVOKE should be queued for the same requestId after redelivery");
+ }
+
+ /**
+ * Case 5 of the #2573 fix: notifyExtensionsOfInvoke()'s own {@code stopped} check, read under
+ * the same lock as quiesce()'s sweep. sendInvocation()'s write is asynchronous, so quiesce()
+ * can run to completion (setting stopped=true and, since this extension is never parked,
+ * queuing its SHUTDOWN straight into pendingEvents) while the runtime write is still in
+ * flight. When that write's onSuccess finally fires, notifyExtensionsOfInvoke() must see
+ * stopped=true and skip, or the extension would receive an INVOKE queued behind the SHUTDOWN
+ * for a container that has already been told to stop. Freezes both sides on the seams the
+ * other quiesce-race tests in this class use, so the write completes only after quiesce's
+ * lock section has already committed stopped=true and the SHUTDOWN offer.
+ */
+ @Test
+ @Timeout(15)
+ void notifyExtensionsOfInvoke_racedByQuiesce_doesNotQueueInvokeBehindShutdown() throws Exception {
+ CountDownLatch writeEntered = new CountDownLatch(1);
+ CountDownLatch releaseWrite = new CountDownLatch(1);
+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);
+ CountDownLatch releaseQuiesce = new CountDownLatch(1);
+
+ server.stop().get(5, TimeUnit.SECONDS);
+ port = findFreePort();
+ server = new RuntimeApiServer(vertx, port) {
+ @Override protected void beforeSendInvocationWrite(String requestId) {
+ writeEntered.countDown();
+ try { releaseWrite.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ @Override protected void afterQuiesceStoppedFlagSet() {
+ quiesceReachedHook.countDown();
+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+ server.start().get(5, TimeUnit.SECONDS);
+
+ // Registered but never polls /event/next, so quiesce() offers SHUTDOWN straight into
+ // pendingEvents (inside its own lock) rather than dispatching to a parked context.
+ String extensionId = registerExtension("adapter", "INVOKE", "SHUTDOWN");
+
+ httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ awaitWaitingContexts(1);
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-race-shutdown", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+ assertTrue(writeEntered.await(5, TimeUnit.SECONDS),
+ "sendInvocation should have reached the write hook");
+
+ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(server::quiesce);
+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),
+ "quiesce should have set stopped=true and queued SHUTDOWN before releasing its lock");
+
+ // Let the runtime write finish now: its onSuccess fires notifyExtensionsOfInvoke() while
+ // quiesce is still frozen just past committing stopped=true.
+ releaseWrite.countDown();
+ Thread.sleep(200);
+ releaseQuiesce.countDown();
+ quiesceDone.get(5, TimeUnit.SECONDS);
+
+ // Only the SHUTDOWN quiesce queued should be there, no INVOKE snuck in behind it.
+ HttpResponse<String> first = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, first.statusCode());
+ assertEquals("SHUTDOWN", new JsonObject(first.body()).getString("eventType"));
+
+ HttpResponse<String> second = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(204, second.statusCode(),
+ "no INVOKE should have been queued behind the SHUTDOWN once the environment stopped");
+ }
+
+ /**
+ * Case 6 of the #2573 fix: pins the one intentional behavior change as deliberate. If the
+ * runtime never polls /next, the extension must never see an INVOKE, matching real AWS, where
+ * delivery is tied to the runtime signaling readiness for the next invocation.
+ */
+ @Test
+ @Timeout(15)
+ void notifyExtensionsOfInvoke_neverFiresIfRuntimeNeverPolls() throws Exception {
+ String extensionId = registerExtension("lambda-adapter", "INVOKE");
+
+ CompletableFuture<HttpResponse<String>> asyncNext = pollExtensionEventNext(extensionId);
+
+ server.enqueue(new PendingInvocation(
+ "req-runtime-never-polls", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>()));
+
+ Thread.sleep(500);
+ assertFalse(asyncNext.isDone(),
+ "extension must not receive INVOKE until the runtime actually polls /next");
+ }
+
@Test
@Timeout(15)
void extensionEventNext_receivesShutdownEventWhenServerStops() throws Exception {
@@ -1768,4 +2067,4 @@ class RuntimeApiServerTest {
}
return server.isExtensionParked(extensionId);
}
-}
+}
\ No newline at end of file