Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): deliver extension INVOKE when the runtime receives the invocation (#3070)
Ammar43e2a482636b
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.javaindex 7349a463b4..da4f2d15d1 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java@@ -725,9 +725,53 @@ public class RuntimeApiServer {return close();}+ /**+ * Fan the INVOKE event out to every subscribed extension. Called from+ * {@link #sendInvocation(RoutingContext, PendingInvocation)}, i.e. at the moment the runtime+ * actually receives the invocation, not at enqueue() time. Nothing in the launch path waits for+ * an internal extension to register, so an extension registering from inside the runtime process+ * may not be in {@code extensions} yet when enqueue() runs; dispatching there dropped the event+ * entirely (see #2573).+ *+ * <p>{@code stopped} is read under the lock for the same reason sendInvocation()'s failure path+ * reads it: quiesce() may already have swept the extensions and queued SHUTDOWN, and an INVOKE+ * offered afterwards would be drained ahead of that SHUTDOWN by /extension/event/next.+ */+ private void notifyExtensionsOfInvoke(PendingInvocation invocation) {+ List<Runnable> dispatches = new ArrayList<>();+ synchronized (lock) {+ if (stopped || extensions.isEmpty()) {+ return;+ }+ ExtensionEvent event = ExtensionEvent.invoke(+ invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());+ for (RegisteredExtension ext : extensions.values()) {+ if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {+ continue;+ }+ RoutingContext waitingCtx = ext.takeWaitingContext();+ if (waitingCtx != null) {+ dispatches.add(() -> {+ if (!waitingCtx.response().ended()) {+ sendExtensionEvent(waitingCtx, event);+ } else {+ synchronized (lock) {+ ext.getPendingEvents().offer(event);+ }+ }+ });+ } else {+ ext.getPendingEvents().offer(event);+ }+ }+ }+ for (Runnable dispatch : dispatches) {+ vertx.runOnContext(v -> dispatch.run());+ }+ }+public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {boolean rejected;- List<Runnable> dispatches = new ArrayList<>();RoutingContext waitingCtxForInvocation = null;synchronized (lock) {@@ -738,30 +782,6 @@ public class RuntimeApiServer {// stays a single atomic step.rejected = stopped || faulted;if (!rejected) {- if (!extensions.isEmpty()) {- ExtensionEvent event = ExtensionEvent.invoke(- invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());- for (RegisteredExtension ext : extensions.values()) {- if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {- continue;- }- RoutingContext waitingCtx = ext.takeWaitingContext();- if (waitingCtx != null) {- dispatches.add(() -> {- if (!waitingCtx.response().ended()) {- sendExtensionEvent(waitingCtx, event);- } else {- synchronized (lock) {- ext.getPendingEvents().offer(event);- }- }- });- } else {- ext.getPendingEvents().offer(event);- }- }- }-waitingCtxForInvocation = waitingContexts.poll();if (waitingCtxForInvocation == null) {pendingQueue.offer(invocation);@@ -780,10 +800,6 @@ public class RuntimeApiServer {return invocation.getResultFuture();}- for (Runnable dispatch : dispatches) {- vertx.runOnContext(v -> dispatch.run());- }-if (waitingCtxForInvocation != null) {final RoutingContext waitingCtx = waitingCtxForInvocation;vertx.runOnContext(v -> {@@ -873,6 +889,9 @@ public class RuntimeApiServer {} catch (IllegalStateException alreadyEnded) {write = Future.failedFuture(alreadyEnded);}+ // Gated on the successful write: onFailure below requeues the invocation, and a fan-out+ // before the write would emit a second INVOKE for the same requestId on redelivery.+ write.onSuccess(v -> notifyExtensionsOfInvoke(invocation));write.onFailure(err -> {// Requeue for the next /next poller; mirrors enqueue()'s ctx-ended branch.// Skip if quiesce() beat us — it already swept inFlight and settled thediff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.javaindex 01e76002cc..c9fdbb8d4b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java@@ -870,6 +870,18 @@ class RuntimeApiServerTest {assertEquals(403, response.statusCode());}+ /**+ * Regression pin for #2573: the extension INVOKE fan-out fires when the runtime actually+ * receives the invocation via /next, not at enqueue() time. Before the fix, enqueue() built+ * and dispatched the event synchronously to whatever was in {@code extensions} at that exact+ * moment, regardless of whether the runtime had polled yet; an internal extension (no+ * {@code /opt/extensions} footprint, so nothing ever waits for its registration) commonly+ * registers concurrently with or after that call and misses the event entirely, with nothing+ * later reconsidering it. The distinguishing assertion is the one right after enqueue():+ * on unfixed code the event is already delivered by that point (enqueue() dispatched it+ * inline), so the test fails there; on fixed code nothing is delivered until /next is+ * actually polled below.+ */@Test@Timeout(15)void extensionEventNext_receivesInvokeEventWhenRuntimeInvocationEnqueued() throws Exception {@@ -890,6 +902,20 @@ class RuntimeApiServerTest {new CompletableFuture<>());server.enqueue(invocation);+ // The #2573 regression check: enqueue() alone must not deliver the INVOKE. Pre-fix,+ // this is exactly where the event arrived (dispatched inline inside enqueue()).+ Thread.sleep(300);+ assertFalse(asyncNext.isDone(),+ "enqueue() alone must not deliver INVOKE; delivery must wait for the runtime's own /next poll");++ // The INVOKE fan-out now happens at sendInvocation() time, so nothing is delivered until+ // the runtime itself polls for the next invocation.+ HttpResponse<String> nextResponse = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, nextResponse.statusCode());+HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);assertEquals(200, response.statusCode());assertTrue(response.headers().firstValue("Lambda-Extension-Event-Identifier").isPresent());@@ -916,11 +942,284 @@ class RuntimeApiServerTest {"arn:aws:lambda:us-east-1:000000000000:function:test",new CompletableFuture<>()));+ // Drive the invocation through to the runtime, same as a real dispatch would, so this+ // exercises the actual fan-out point rather than a poller that never arrives.+ httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+Thread.sleep(500);assertFalse(asyncNext.isDone(),"extension not subscribed to INVOKE must not be woken by an invocation");}+ /**+ * Case 3 of the #2573 fix: an extension that has registered but not yet issued its first+ * /event/next when the invocation dispatches must still receive it. notifyExtensionsOfInvoke()+ * offers the event into the extension's pendingEvents queue rather than dropping it when+ * there is no parked context to write to directly.+ */+ @Test+ @Timeout(15)+ void notifyExtensionsOfInvoke_extensionRegisteredButNotYetPolling_queuesEventForLaterPoll()+ throws Exception {+ String extensionId = registerExtension("lambda-adapter", "INVOKE");++ PendingInvocation invocation = new PendingInvocation(+ "req-not-yet-polling", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, next.statusCode());++ // The extension's first poll arrives after the dispatch already happened, so it must find+ // the event waiting rather than parking forever.+ HttpResponse<String> eventResponse = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, eventResponse.statusCode());+ JsonObject body = new JsonObject(eventResponse.body());+ assertEquals("INVOKE", body.getString("eventType"));+ assertEquals("req-not-yet-polling", body.getString("requestId"));+ }++ /**+ * Case 4 of the #2573 fix: the common warm-container path, where the runtime is already+ * parked on /next and the extension already parked on /event/next when enqueue() runs.+ * Exercises the deferred-dispatch branch (enqueue()'s vertx.runOnContext callback) rather+ * than the synchronous NEXT_PATH handler, and confirms relocating the fan-out to+ * sendInvocation() did not regress this case.+ */+ @Test+ @Timeout(15)+ void notifyExtensionsOfInvoke_warmPath_runtimeAndExtensionBothParked_deliversBoth()+ throws Exception {+ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ String extensionId = registerExtension("lambda-adapter", "INVOKE");+ CompletableFuture<HttpResponse<String>> asyncExtensionNext = pollExtensionEventNext(extensionId);++ PendingInvocation invocation = new PendingInvocation(+ "req-warm-path", "{\"warm\":true}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ HttpResponse<String> runtimeResponse = asyncNext.get(2, TimeUnit.SECONDS);+ assertEquals(200, runtimeResponse.statusCode());+ assertEquals("req-warm-path",+ runtimeResponse.headers().firstValue("Lambda-Runtime-Aws-Request-Id").orElse(""));++ HttpResponse<String> extensionResponse = asyncExtensionNext.get(2, TimeUnit.SECONDS);+ assertEquals(200, extensionResponse.statusCode());+ JsonObject body = new JsonObject(extensionResponse.body());+ assertEquals("INVOKE", body.getString("eventType"));+ assertEquals("req-warm-path", body.getString("requestId"));+ }++ /**+ * Case 2 of the #2573 fix, and the maintainer's stated requirement: sendInvocation()'s+ * onFailure requeues the invocation for redelivery to a second /next poller. Since+ * notifyExtensionsOfInvoke() is gated on the write's onSuccess, the failed first attempt must+ * not have fired it. Only the second, successful write should, so the extension sees exactly+ * one INVOKE for the requestId, never two. Forces the first write to fail the same way+ * {@link #sendInvocation_writeFails_requeuesAndClearsInFlight} does, then counts+ * beforeSendInvocationWrite calls to prove the redelivery actually happened rather than the+ * assertions passing by coincidence.+ */+ @Test+ @Timeout(15)+ void notifyExtensionsOfInvoke_requeuedOnWriteFailure_deliversInvokeExactlyOnce() throws Exception {+ java.util.concurrent.atomic.AtomicInteger sendInvocationCalls =+ new java.util.concurrent.atomic.AtomicInteger();+ java.util.concurrent.atomic.AtomicReference<io.vertx.ext.web.RoutingContext> parkedCtx =+ new java.util.concurrent.atomic.AtomicReference<>();+ server.stop().get(5, TimeUnit.SECONDS);+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port) {+ @Override protected void afterEnqueueDispatchLockReleased(io.vertx.ext.web.RoutingContext waitingCtx) {+ parkedCtx.set(waitingCtx);+ }+ @Override protected void beforeSendInvocationWrite(String requestId) {+ if (sendInvocationCalls.incrementAndGet() == 1) {+ // Simulate the client disconnecting between dispatch commitment and the+ // write landing, forcing sendInvocation()'s write to fail.+ io.vertx.ext.web.RoutingContext ctx = parkedCtx.get();+ if (ctx != null && !ctx.response().ended()) {+ ctx.response().setStatusCode(500).end();+ }+ }+ }+ };+ server.start().get(5, TimeUnit.SECONDS);++ String extensionId = registerExtension("lambda-adapter", "INVOKE");+ CompletableFuture<HttpResponse<String>> asyncExtensionNext = pollExtensionEventNext(extensionId);++ httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ PendingInvocation invocation = new PendingInvocation(+ "req-no-duplicate", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);++ // Wait for the forced-failed dispatch to requeue.+ long deadline = System.currentTimeMillis() + 5000;+ while (System.currentTimeMillis() < deadline && server.pendingQueueSize() == 0) {+ Thread.sleep(10);+ }+ assertEquals(1, server.pendingQueueSize(),+ "invocation must be requeued after the forced write failure");++ // A second poller picks up the requeued invocation; this write succeeds.+ HttpResponse<String> secondNext = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, secondNext.statusCode());++ assertEquals(2, sendInvocationCalls.get(),+ "sendInvocation must have been attempted twice: the failed original, and the redelivery");++ HttpResponse<String> extensionResponse = asyncExtensionNext.get(2, TimeUnit.SECONDS);+ assertEquals(200, extensionResponse.statusCode());+ assertEquals("req-no-duplicate", new JsonObject(extensionResponse.body()).getString("requestId"));++ // No second INVOKE should be waiting behind the one just delivered.+ CompletableFuture<HttpResponse<String>> secondExtensionPoll = httpClient.sendAsync(+ HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ Thread.sleep(300);+ assertFalse(secondExtensionPoll.isDone(),+ "no duplicate INVOKE should be queued for the same requestId after redelivery");+ }++ /**+ * Case 5 of the #2573 fix: notifyExtensionsOfInvoke()'s own {@code stopped} check, read under+ * the same lock as quiesce()'s sweep. sendInvocation()'s write is asynchronous, so quiesce()+ * can run to completion (setting stopped=true and, since this extension is never parked,+ * queuing its SHUTDOWN straight into pendingEvents) while the runtime write is still in+ * flight. When that write's onSuccess finally fires, notifyExtensionsOfInvoke() must see+ * stopped=true and skip, or the extension would receive an INVOKE queued behind the SHUTDOWN+ * for a container that has already been told to stop. Freezes both sides on the seams the+ * other quiesce-race tests in this class use, so the write completes only after quiesce's+ * lock section has already committed stopped=true and the SHUTDOWN offer.+ */+ @Test+ @Timeout(15)+ void notifyExtensionsOfInvoke_racedByQuiesce_doesNotQueueInvokeBehindShutdown() throws Exception {+ CountDownLatch writeEntered = new CountDownLatch(1);+ CountDownLatch releaseWrite = new CountDownLatch(1);+ CountDownLatch quiesceReachedHook = new CountDownLatch(1);+ CountDownLatch releaseQuiesce = new CountDownLatch(1);++ server.stop().get(5, TimeUnit.SECONDS);+ port = findFreePort();+ server = new RuntimeApiServer(vertx, port) {+ @Override protected void beforeSendInvocationWrite(String requestId) {+ writeEntered.countDown();+ try { releaseWrite.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ @Override protected void afterQuiesceStoppedFlagSet() {+ quiesceReachedHook.countDown();+ try { releaseQuiesce.await(5, TimeUnit.SECONDS); } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ };+ server.start().get(5, TimeUnit.SECONDS);++ // Registered but never polls /event/next, so quiesce() offers SHUTDOWN straight into+ // pendingEvents (inside its own lock) rather than dispatching to a parked context.+ String extensionId = registerExtension("adapter", "INVOKE", "SHUTDOWN");++ httpClient.sendAsync(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ awaitWaitingContexts(1);++ PendingInvocation invocation = new PendingInvocation(+ "req-race-shutdown", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>());+ server.enqueue(invocation);+ assertTrue(writeEntered.await(5, TimeUnit.SECONDS),+ "sendInvocation should have reached the write hook");++ CompletableFuture<Void> quiesceDone = CompletableFuture.runAsync(server::quiesce);+ assertTrue(quiesceReachedHook.await(5, TimeUnit.SECONDS),+ "quiesce should have set stopped=true and queued SHUTDOWN before releasing its lock");++ // Let the runtime write finish now: its onSuccess fires notifyExtensionsOfInvoke() while+ // quiesce is still frozen just past committing stopped=true.+ releaseWrite.countDown();+ Thread.sleep(200);+ releaseQuiesce.countDown();+ quiesceDone.get(5, TimeUnit.SECONDS);++ // Only the SHUTDOWN quiesce queued should be there, no INVOKE snuck in behind it.+ HttpResponse<String> first = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(200, first.statusCode());+ assertEquals("SHUTDOWN", new JsonObject(first.body()).getString("eventType"));++ HttpResponse<String> second = httpClient.send(HttpRequest.newBuilder()+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))+ .header("Lambda-Extension-Identifier", extensionId)+ .GET().build(),+ HttpResponse.BodyHandlers.ofString());+ assertEquals(204, second.statusCode(),+ "no INVOKE should have been queued behind the SHUTDOWN once the environment stopped");+ }++ /**+ * Case 6 of the #2573 fix: pins the one intentional behavior change as deliberate. If the+ * runtime never polls /next, the extension must never see an INVOKE, matching real AWS, where+ * delivery is tied to the runtime signaling readiness for the next invocation.+ */+ @Test+ @Timeout(15)+ void notifyExtensionsOfInvoke_neverFiresIfRuntimeNeverPolls() throws Exception {+ String extensionId = registerExtension("lambda-adapter", "INVOKE");++ CompletableFuture<HttpResponse<String>> asyncNext = pollExtensionEventNext(extensionId);++ server.enqueue(new PendingInvocation(+ "req-runtime-never-polls", "{}".getBytes(), System.currentTimeMillis() + 60_000,+ "arn:aws:lambda:us-east-1:000000000000:function:test",+ new CompletableFuture<>()));++ Thread.sleep(500);+ assertFalse(asyncNext.isDone(),+ "extension must not receive INVOKE until the runtime actually polls /next");+ }+@Test@Timeout(15)void extensionEventNext_receivesShutdownEventWhenServerStops() throws Exception {@@ -1768,4 +2067,4 @@ class RuntimeApiServerTest {}return server.isExtensionParked(extensionId);}-}+}\ No newline at end of file