Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): honour the Invoke Qualifier and wait between asynchronous retries as AWS does (#4733)
Mario Toffiafa4d4663a24b
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaAsyncInvocationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaAsyncInvocationTest.javanew file mode 100644index 0000000000..ceca6ec39e--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaAsyncInvocationTest.java@@ -0,0 +1,147 @@+package com.floci.test;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.core.SdkBytes;+import software.amazon.awssdk.services.lambda.LambdaClient;+import software.amazon.awssdk.services.lambda.model.CreateAliasRequest;+import software.amazon.awssdk.services.lambda.model.CreateFunctionRequest;+import software.amazon.awssdk.services.lambda.model.DeleteFunctionRequest;+import software.amazon.awssdk.services.lambda.model.DestinationConfig;+import software.amazon.awssdk.services.lambda.model.FunctionCode;+import software.amazon.awssdk.services.lambda.model.InvocationType;+import software.amazon.awssdk.services.lambda.model.InvokeRequest;+import software.amazon.awssdk.services.lambda.model.InvokeResponse;+import software.amazon.awssdk.services.lambda.model.OnFailure;+import software.amazon.awssdk.services.lambda.model.PublishVersionRequest;+import software.amazon.awssdk.services.lambda.model.PutFunctionEventInvokeConfigRequest;+import software.amazon.awssdk.services.lambda.model.Runtime;+import software.amazon.awssdk.services.sqs.SqsClient;+import software.amazon.awssdk.services.sqs.model.CreateQueueRequest;+import software.amazon.awssdk.services.sqs.model.DeleteQueueRequest;+import software.amazon.awssdk.services.sqs.model.GetQueueAttributesRequest;+import software.amazon.awssdk.services.sqs.model.Message;+import software.amazon.awssdk.services.sqs.model.QueueAttributeName;+import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest;++import java.time.Duration;+import java.time.Instant;+import java.util.List;++import static org.assertj.core.api.Assertions.assertThat;++@DisplayName("Lambda asynchronous invocation")+class LambdaAsyncInvocationTest {++ private static final String ROLE = "arn:aws:iam::000000000000:role/lambda-role";+ private static final Duration RECEIVE_DEADLINE = Duration.ofSeconds(60);+ private static final ObjectMapper MAPPER = new ObjectMapper();++ private static LambdaClient lambda;+ private static SqsClient sqs;+ private static String functionName;+ private static String queueUrl;++ @BeforeAll+ static void setup() {+ lambda = TestFixtures.lambdaClient();+ sqs = TestFixtures.sqsClient();+ }++ @AfterAll+ static void cleanup() {+ if (lambda != null) {+ if (functionName != null) {+ try {+ lambda.deleteFunction(DeleteFunctionRequest.builder().functionName(functionName).build());+ } catch (Exception e) {+ System.err.println("Best-effort Lambda async invocation test cleanup failed: " + e.getMessage());+ }+ }+ lambda.close();+ }+ if (sqs != null) {+ if (queueUrl != null) {+ try {+ sqs.deleteQueue(DeleteQueueRequest.builder().queueUrl(queueUrl).build());+ } catch (Exception e) {+ System.err.println("Best-effort SQS async invocation test cleanup failed: " + e.getMessage());+ }+ }+ sqs.close();+ }+ }++ @Test+ @DisplayName("Invoke with Qualifier applies the alias's OnFailure destination")+ void invokeWithQualifierAppliesTheAliasOnFailureDestination() throws Exception {+ queueUrl = sqs.createQueue(CreateQueueRequest.builder()+ .queueName(TestFixtures.uniqueName("sdk-async-onfailure"))+ .build()).queueUrl();+ String queueArn = sqs.getQueueAttributes(GetQueueAttributesRequest.builder()+ .queueUrl(queueUrl)+ .attributeNames(QueueAttributeName.QUEUE_ARN)+ .build()).attributes().get(QueueAttributeName.QUEUE_ARN);++ functionName = TestFixtures.uniqueName("sdk-async-fn");+ lambda.createFunction(CreateFunctionRequest.builder()+ .functionName(functionName)+ .runtime(Runtime.NODEJS20_X)+ .role(ROLE)+ .handler("index.handler")+ .timeout(10)+ .code(FunctionCode.builder()+ .zipFile(SdkBytes.fromByteArray(LambdaUtils.failingZip()))+ .build())+ .build());+ String version = lambda.publishVersion(PublishVersionRequest.builder()+ .functionName(functionName)+ .build()).version();+ lambda.createAlias(CreateAliasRequest.builder()+ .functionName(functionName)+ .name("live")+ .functionVersion(version)+ .build());+ lambda.putFunctionEventInvokeConfig(PutFunctionEventInvokeConfigRequest.builder()+ .functionName(functionName)+ .qualifier("live")+ .maximumRetryAttempts(0)+ .destinationConfig(DestinationConfig.builder()+ .onFailure(OnFailure.builder().destination(queueArn).build())+ .build())+ .build());++ InvokeResponse response = lambda.invoke(InvokeRequest.builder()+ .functionName(functionName)+ .qualifier("live")+ .invocationType(InvocationType.EVENT)+ .payload(SdkBytes.fromUtf8String("{\"probe\":\"sdk\"}"))+ .build());+ assertThat(response.statusCode()).isEqualTo(202);++ JsonNode record = MAPPER.readTree(awaitMessage().body());+ assertThat(record.path("requestContext").path("approximateInvokeCount").asInt()).isEqualTo(1);+ assertThat(record.path("requestContext").path("condition").asText()).isEqualTo("RetriesExhausted");+ assertThat(record.path("requestPayload").path("probe").asText()).isEqualTo("sdk");+ assertThat(record.path("responseContext").path("executedVersion").asText()).isEqualTo(version);+ }++ private static Message awaitMessage() {+ Instant deadline = Instant.now().plus(RECEIVE_DEADLINE);+ while (Instant.now().isBefore(deadline)) {+ List<Message> messages = sqs.receiveMessage(ReceiveMessageRequest.builder()+ .queueUrl(queueUrl)+ .waitTimeSeconds(20)+ .maxNumberOfMessages(1)+ .build()).messages();+ if (!messages.isEmpty()) {+ return messages.get(0);+ }+ }+ throw new AssertionError("no OnFailure record arrived within " + RECEIVE_DEADLINE.toSeconds() + " s");+ }+}diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.javaindex d8ea6abf1a..4cf9af8c9e 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java@@ -203,14 +203,9 @@ class LambdaConcurrencyTest {@Test@Order(12)void invoke_withVersionQualifier_stillHonorsReservedOnLatest() {- // Regression guard: if a future change adds Qualifier routing that- // resolves to a published version snapshot, the snapshot currently- // has reservedConcurrentExecutions=null and would silently bypass a- // reserved=0 on $LATEST. Today Floci ignores the qualifier and- // routes the invoke to $LATEST, so reserved=0 must still throttle.- // Keeping this test green after a qualifier-routing change will- // require copying the reservation onto the snapshot (or keying the- // limiter off the base ARN).+ // Reserved concurrency is function-wide: Qualifier routes the invoke to+ // the published version, whose snapshot carries no reservation, and the+ // function's reserved=0 must still throttle it.lambda.publishVersion(PublishVersionRequest.builder().functionName(FUNCTION_NAME).build());lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.javaindex fa7de92808..fe88ca375b 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaUtils.java@@ -109,6 +109,16 @@ public final class LambdaUtils {return createZip("index.js", code);}+ /**+ * ZIP containing a Node.js handler that always throws, for asynchronous failure handling tests.+ */+ public static byte[] failingZip() {+ String code = """+ exports.handler = async () => { throw new Error('boom'); };+ """;+ return createZip("index.js", code);+ }+/*** Minimal valid ZIP containing a stub index.js.*/diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex 0f74feffe5..b1c786c755 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -163,6 +163,7 @@ floci:code-path: ./data/lambda-code # Where ZIP archives are storedzip-max-entries: 100000 # Maximum ZIP entries extracted per deployment packagepoll-interval-ms: 1000+ async-retry-delay-seconds: 60 # Wait before the first retry of a failed Event invocation; the second waits twice thiscontainer-idle-timeout-seconds: 300 # Remove idle containers after thisregion-concurrency-limit: 1000 # Concurrent executions ceiling per regionunreserved-concurrency-min: 100 # Minimum unreserved capacity PutFunctionConcurrency must leavediff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex 1037c54177..01ca1d3d50 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -244,6 +244,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_LAMBDA_RUNTIME_API_MAX_PORT` | `12499` | Last port in the Lambda Runtime API port range. One port is held per running Lambda container, so the range width is the concurrent-execution ceiling || `FLOCI_SERVICES_LAMBDA_CODE_PATH` | `./data/lambda-code` | Container path where Lambda deployment ZIPs are stored || `FLOCI_SERVICES_LAMBDA_POLL_INTERVAL_MS` | `1000` | How often (ms) the SQS and Kinesis event source pollers check for new messages |+| `FLOCI_SERVICES_LAMBDA_ASYNC_RETRY_DELAY_SECONDS` | `60` | Wait before the first retry of a failed asynchronous invocation (seconds); the second waits twice this, `0` retries back to back || `FLOCI_SERVICES_LAMBDA_CONTAINER_IDLE_TIMEOUT_SECONDS` | `300` | Seconds of inactivity before an idle Lambda container is removed || `FLOCI_SERVICES_LAMBDA_REGION_CONCURRENCY_LIMIT` | `1000` | Maximum concurrent Lambda invocations across all functions in a region || `FLOCI_SERVICES_LAMBDA_UNRESERVED_CONCURRENCY_MIN` | `100` | Minimum unreserved concurrency pool |diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex 5b1ceb1c01..c57e553cfb 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -77,9 +77,12 @@ subsequent unqualified `GetFunction` responses.An asynchronous invocation (`InvocationType: Event`) answers `202` immediately and runs on the backgroundpool. If the invocation fails, it is retried up to `MaximumRetryAttempts` (default: 2, total 3 attempts)-unless the event age exceeds `MaximumEventAgeInSeconds` (default: 21600 seconds, 6 hours). In Floci,-retries run back to back without the delays of live AWS (which waits roughly one minute and then two) to keep-local test runs fast.+unless the event age exceeds `MaximumEventAgeInSeconds` (default: 21600 seconds, 6 hours). As in AWS, the+first retry waits one minute and the second two. `FLOCI_SERVICES_LAMBDA_ASYNC_RETRY_DELAY_SECONDS` sets the+first wait (the second is twice it), and `0` retries back to back for fast local test runs. A wait never runs+past `MaximumEventAgeInSeconds`, so an event that expires while waiting is reported then. A retry that finds the+function's concurrency in use tries again every second until the event expires, where AWS backs off for up to+five minutes. Deleting the function drops its pending retries without delivering a record, as AWS does.When retries are exhausted or event age is exceeded:@@ -125,8 +128,9 @@ type `Lambda Function Invocation Result - Failure`, a `condition` of `RetriesExh`functionError` member is present only on a failure record. The `approximateInvokeCount` field reflectsthe actual number of attempts executed before completion or exhaustion.-An alias-specific destination configuration is used when the alias is invoked. If the alias has-none, Floci checks the configuration on the version that the alias resolves to.+An alias-specific destination configuration is used when the alias is invoked, whether the alias is named+in `Qualifier` or on the function name. If the alias has none, Floci checks the configuration on the+version that the alias resolves to.Two limits are worth knowing:@@ -257,14 +261,16 @@ disabled fails with Lambda's error as the resource status reason.| Path on host required | No | Yes |!!! note "Concurrency enforcement"- Reserved concurrency is enforced: invocations beyond the reserved value- return `TooManyRequestsException` (HTTP 429). Functions without a reserved- value share a **per-region** pool. AWS Lambda's "account-level" limit is- in fact a per-account-per-region quota, and Floci mirrors that by- partitioning counters on the ARN's region segment. The pool size (default- 1000) is configurable via `floci.services.lambda.region-concurrency-limit`- and applies independently to each region. `PutFunctionConcurrency`- validates that the requested value leaves at least+ Reserved concurrency is enforced function-wide: invocations of every+ version and alias count against the function's reservation, and+ invocations beyond the reserved value return `TooManyRequestsException`+ (HTTP 429). Functions without a reserved value share a **per-region**+ pool. AWS Lambda's "account-level" limit is in fact a+ per-account-per-region quota, and Floci mirrors that by partitioning+ counters on the ARN's region segment. The pool size (default 1000) is+ configurable via `floci.services.lambda.region-concurrency-limit` and+ applies independently to each region. `PutFunctionConcurrency` validates+ that the requested value leaves at least`floci.services.lambda.unreserved-concurrency-min` (default 100) availablefor unreserved functions in that region. `PutProvisionedConcurrencyConfig`and related provisioned-concurrency operations remain unimplemented.@@ -367,6 +373,7 @@ These AWS Lambda operations have no handler in Floci. Calls will return `404` or| `FLOCI_SERVICES_LAMBDA_RUNTIME_API_MAX_PORT` | `12499` | Last port in the Lambda Runtime API range. One port is held per running container, so the range width caps concurrent executions. Each running container also holds two Docker connections, so a wider range needs a matching `FLOCI_DOCKER_MAX_CONNECTIONS` || `FLOCI_SERVICES_LAMBDA_CODE_PATH` | `./data/lambda-code` | Directory where Lambda ZIP files are stored || `FLOCI_SERVICES_LAMBDA_POLL_INTERVAL_MS` | `1000` | Event-source mapping poll interval (milliseconds) |+| `FLOCI_SERVICES_LAMBDA_ASYNC_RETRY_DELAY_SECONDS` | `60` | Wait before the first retry of a failed asynchronous invocation (seconds); the second waits twice this, `0` retries back to back || `FLOCI_SERVICES_LAMBDA_CONTAINER_IDLE_TIMEOUT_SECONDS` | `300` | Idle container shutdown timeout (seconds) || `FLOCI_SERVICES_LAMBDA_REGION_CONCURRENCY_LIMIT` | `1000` | Maximum concurrent executions per region || `FLOCI_SERVICES_LAMBDA_UNRESERVED_CONCURRENCY_MIN` | `100` | Minimum unreserved capacity `PutFunctionConcurrency` must leave |diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 543d1273d5..dcb71e2082 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -2733,6 +2733,13 @@ public interface EmulatorConfig {@WithDefault("1000")long pollIntervalMs();+ /**+ * Seconds before the first retry of a failed asynchronous invocation. Retry n waits n times+ * this, so the default of 60 matches AWS: one minute, then two. Zero retries back to back.+ */+ @WithDefault("60")+ int asyncRetryDelaySeconds();+@WithDefault("false")boolean ephemeral();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.javaindex 98692799f2..67b20a2162 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java@@ -238,8 +238,12 @@ public class LambdaController {@Path("/functions/{functionName}/invocations")@Consumes(MediaType.WILDCARD)public Response invoke(@Context HttpHeaders headers,+ @Context UriInfo uriInfo,@PathParam("functionName") String functionName,byte[] payload) {+ // Read from the query itself: @QueryParam reports a present but empty Qualifier as null, which+ // would hide it from the Qualifier validation.+ String qualifier = uriInfo.getQueryParameters().getFirst("Qualifier");String region = regionResolver.resolveRegion(headers);String invocationTypeHeader = headers.getHeaderString("X-Amz-Invocation-Type");InvocationType type = InvocationType.parse(invocationTypeHeader);@@ -258,7 +262,7 @@ public class LambdaController {.build();}- InvokeResult result = lambdaService.invoke(region, functionName, payload, type);+ InvokeResult result = lambdaService.invoke(region, functionName, qualifier, payload, type);if (type != InvocationType.Event&& result.getPayload() != nulldiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex 0b89071bdf..867a74050b 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -2,6 +2,10 @@ package io.github.hectorvent.floci.services.lambda;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.model.FunctionEventInvokeConfig;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;@@ -15,28 +19,43 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.time.Clock;+import java.time.Duration;+import java.util.Map;import java.util.UUID;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutionException;+import java.util.concurrent.Executor;import java.util.concurrent.ExecutorService;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicLong;/*** Orchestrates Lambda function invocations.* Handles RequestResponse (sync), Event (async fire-and-forget), and DryRun modes.*- * <p>An Event invocation answers 202 straight away and, once the function has finished on the- * pool, hands its result to {@link AsyncInvokeDestinationRouter}, which delivers it to the- * function's configured destination when it has one.+ * <p>An Event invocation answers 202 straight away and hands the final outcome to+ * {@link AsyncInvokeDestinationRouter}, which delivers it to the function's configured destination+ * when it has one. The first attempt runs on the pool; a failed attempt is retried on a virtual+ * thread after the configured delay, as AWS waits one minute and then two.*/@ApplicationScoped-public class LambdaExecutorService {+public class LambdaExecutorService implements Resettable {private static final Logger LOG = Logger.getLogger(LambdaExecutorService.class);/** Extra time for a newly started runtime to request its first invocation. */private static final int RUNTIME_DISPATCH_GRACE_SECONDS = 2;+ /** How long a retry that found the function's concurrency in use waits before asking again. */+ private static final long THROTTLED_RETRY_POLL_MS = 1000;+ /**+ * Retries run on virtual threads rather than the pool: once the pool's queue is full its+ * caller-runs fallback would run a whole attempt on the JDK's shared delay thread, which also+ * fires the invocation timeouts that attempt may be waiting on.+ */+ private static final Executor RETRY_THREADS = task -> Thread.ofVirtual().name("lambda-async-retry").start(task);private final WarmPool warmPool;private final ObjectMapper objectMapper;@@ -46,6 +65,10 @@ public class LambdaExecutorService {private final Instance<LambdaService> lambdaServiceInstance;private final LambdaService directLambdaService;private final Clock clock;+ private final long asyncRetryDelayMs;+ /** Moved on by a reset so that retries still waiting from before it are dropped. */+ private final AtomicLong generation = new AtomicLong();+ private final Map<String, Long> deletions = new ConcurrentHashMap<>();private final ExecutorService asyncExecutor = new ThreadPoolExecutor(Math.max(4, Runtime.getRuntime().availableProcessors() * 2),Math.max(8, Runtime.getRuntime().availableProcessors() * 4),@@ -59,8 +82,10 @@ public class LambdaExecutorService {LambdaConcurrencyLimiter concurrencyLimiter,AsyncInvokeDestinationRouter destinationRouter,Instance<LambdaService> lambdaServiceInstance,- Clock clock) {- this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, lambdaServiceInstance, null, clock);+ Clock clock,+ EmulatorConfig config) {+ this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, lambdaServiceInstance, null, clock,+ Duration.ofSeconds(config.services().lambda().asyncRetryDelaySeconds()));}LambdaExecutorService(WarmPool warmPool,@@ -69,7 +94,7 @@ public class LambdaExecutorService {AsyncInvokeDestinationRouter destinationRouter,LambdaService directLambdaService) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService,- Clock.systemUTC());+ Clock.systemUTC(), Duration.ZERO);}LambdaExecutorService(WarmPool warmPool,@@ -77,8 +102,10 @@ public class LambdaExecutorService {LambdaConcurrencyLimiter concurrencyLimiter,AsyncInvokeDestinationRouter destinationRouter,LambdaService directLambdaService,- Clock clock) {- this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService, clock);+ Clock clock,+ Duration asyncRetryDelay) {+ this(warmPool, objectMapper, concurrencyLimiter, destinationRouter, null, directLambdaService, clock,+ asyncRetryDelay);}private LambdaExecutorService(WarmPool warmPool,@@ -87,7 +114,8 @@ public class LambdaExecutorService {AsyncInvokeDestinationRouter destinationRouter,Instance<LambdaService> lambdaServiceInstance,LambdaService directLambdaService,- Clock clock) {+ Clock clock,+ Duration asyncRetryDelay) {this.warmPool = warmPool;this.objectMapper = objectMapper;this.concurrencyLimiter = concurrencyLimiter;@@ -95,6 +123,7 @@ public class LambdaExecutorService {this.lambdaServiceInstance = lambdaServiceInstance;this.directLambdaService = directLambdaService;this.clock = clock;+ this.asyncRetryDelayMs = asyncRetryDelay.toMillis();}/** Package-private constructor for testing without CDI, leaving destinations unrouted. */@@ -102,7 +131,7 @@ public class LambdaExecutorService {ObjectMapper objectMapper,LambdaConcurrencyLimiter concurrencyLimiter) {this(warmPool, objectMapper, concurrencyLimiter, null, (Instance<LambdaService>) null, null,- Clock.systemUTC());+ Clock.systemUTC(), Duration.ZERO);}LambdaExecutorService(WarmPool warmPool,@@ -110,7 +139,7 @@ public class LambdaExecutorService {LambdaConcurrencyLimiter concurrencyLimiter,AsyncInvokeDestinationRouter destinationRouter) {this(warmPool, objectMapper, concurrencyLimiter, destinationRouter,- (Instance<LambdaService>) null, null, Clock.systemUTC());+ (Instance<LambdaService>) null, null, Clock.systemUTC(), Duration.ZERO);}public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type) {@@ -146,10 +175,10 @@ public class LambdaExecutorService {return new InvokeResult(202, null, new byte[0], null, requestId);}- LambdaConcurrencyLimiter.Permit permit = concurrencyLimiter.acquire(fn);+ LambdaService lambdaService = resolveLambdaService();+ LambdaConcurrencyLimiter.Permit permit = acquire(lambdaService, fn);if (type == InvocationType.Event) {- LambdaService lambdaService = resolveLambdaService();FunctionEventInvokeConfig eventInvokeConfig = null;if (lambdaService != null) {try {@@ -165,46 +194,19 @@ public class LambdaExecutorService {int maxEventAgeSeconds = eventInvokeConfig != null&& eventInvokeConfig.getMaximumEventAgeInSeconds() != null? eventInvokeConfig.getMaximumEventAgeInSeconds() : 21600;- long submitTimeMs = clock.millis();+ AsyncEvent event = new AsyncEvent(fn, payload, requestId, chainDepth, invokedQualifier,+ maxRetries, clock.millis() + maxEventAgeSeconds * 1000L, generation.get(),+ deletions.getOrDefault(functionKey(fn), 0L));+ // The deletion count is read above, before this store check, and DeleteFunction moves the count+ // again after it removes the function from the store. So either this check sees the function+ // gone, or the count moves after the event's snapshot and stale() drops the event.+ if (lambdaService != null && !lambdaService.isLive(fn)) {+ permit.close();+ return new InvokeResult(202, null, new byte[0], null, requestId);+ }try {- asyncExecutor.submit(() -> {- int attempt = 0;- InvokeResult asyncResult = null;- try {- while (attempt <= maxRetries) {- long elapsedSeconds = (clock.millis() - submitTimeMs) / 1000;- if (elapsedSeconds >= maxEventAgeSeconds) {- break;- }-- attempt++;- asyncResult = executeSync(fn, payload, requestId);- if (asyncResult.getFunctionError() == null && asyncResult.getStatusCode() < 300) {- break;- }- }- } catch (Exception e) {- LOG.warnv("Error in async Lambda execution for {0}: {1}", fn.getFunctionName(), e.getMessage());- if (asyncResult == null) {- asyncResult = new InvokeResult(500, "Unhandled",- buildErrorPayload("Error executing Lambda: " + e.getMessage(), "Lambda.UnknownError"),- null, requestId);- }- } finally {- permit.close();- }- if (destinationRouter != null) {- if (asyncResult == null) {- // This Floci-only placeholder covers expiry before any attempt; AWS documents no payload.- asyncResult = new InvokeResult(200, "Unhandled",- buildErrorPayload("Event age exceeded", "EventAgeExceeded"),- null, requestId);- }- destinationRouter.route(fn, payload, asyncResult, attempt,- chainDepth, invokedQualifier);- }- });+ asyncExecutor.submit(() -> attempt(event, 1, null, permit));} catch (RuntimeException e) {permit.close();throw e;@@ -229,6 +231,87 @@ public class LambdaExecutorService {return null;}+ /**+ * Reserved concurrency is function-wide, so the permit is taken against the function's {@code $LATEST}+ * record, which holds the reservation, rather than a version's snapshot, which has none.+ */+ private LambdaConcurrencyLimiter.Permit acquire(LambdaService lambdaService, LambdaFunction fn) {+ LambdaFunction function = lambdaService != null ? lambdaService.findLatest(fn).orElse(fn) : fn;+ return concurrencyLimiter.acquire(function);+ }++ /**+ * Runs one attempt of an asynchronous event and either routes the outcome or schedules the next+ * attempt. Retry n waits n times the configured delay, never past the event's maximum age, and+ * the pending event of a function deleted meanwhile is dropped, whether it was waiting to retry+ * or to expire. The concurrency permit covers only the attempt; {@code held} is the one the+ * invoke itself took.+ */+ private void attempt(AsyncEvent event, int attempt, InvokeResult previous,+ LambdaConcurrencyLimiter.Permit held) {+ if (stale(event) || clock.millis() >= event.expiresAtMs()) {+ if (held != null) {+ held.close();+ }+ route(event, previous, attempt - 1);+ return;+ }+ LambdaConcurrencyLimiter.Permit permit;+ if (held != null) {+ permit = held;+ } else {+ try {+ permit = acquire(resolveLambdaService(), event.fn());+ } catch (AwsException throttled) {+ // ponytail: AWS backs a throttled retry off exponentially, up to five minutes; one+ // fixed poll is enough locally, and the event still expires on time.+ schedule(event, THROTTLED_RETRY_POLL_MS, attempt, previous);+ return;+ }+ }+ InvokeResult result;+ try (permit) {+ result = executeSync(event.fn(), event.payload(), event.requestId());+ } catch (RuntimeException e) {+ LOG.warnv("Error in async Lambda execution for {0}: {1}", event.fn().getFunctionName(), e.getMessage());+ // Stops retrying, as upstream did: routes the previous attempt's result, or UnknownError when+ // there is none, counting this attempt.+ route(event, previous != null ? previous : new InvokeResult(500, "Unhandled",+ buildErrorPayload("Error executing Lambda: " + e.getMessage(), "Lambda.UnknownError"),+ null, event.requestId()), attempt);+ return;+ }+ if ((result.getFunctionError() == null && result.getStatusCode() < 300) || attempt > event.maxRetries()) {+ route(event, result, attempt);+ return;+ }+ schedule(event, asyncRetryDelayMs * attempt, attempt + 1, result);+ }++ /**+ * Decides expiry up front, on the injected clock: the delay thread keeps its own time and may+ * wake a hair early.+ */+ private void schedule(AsyncEvent event, long delayMs, int attempt, InvokeResult previous) {+ long remainingMs = event.expiresAtMs() - clock.millis();+ Runnable next = delayMs < remainingMs+ ? () -> attempt(event, attempt, previous, null)+ : () -> route(event, previous, attempt - 1);+ CompletableFuture.delayedExecutor(Math.max(0, Math.min(delayMs, remainingMs)), TimeUnit.MILLISECONDS,+ RETRY_THREADS).execute(next);+ }++ private void route(AsyncEvent event, InvokeResult result, int attempts) {+ if (destinationRouter == null || stale(event)) {+ return;+ }+ // This Floci-only placeholder covers expiry before any attempt; AWS documents no payload.+ InvokeResult outcome = result != null ? result : new InvokeResult(200, "Unhandled",+ buildErrorPayload("Event age exceeded", "EventAgeExceeded"), null, event.requestId());+ destinationRouter.route(event.fn(), event.payload(), outcome, attempts, event.chainDepth(),+ event.invokedQualifier());+ }+private InvokeResult executeSync(LambdaFunction fn, byte[] payload, String requestId) {ContainerHandle handle;try {@@ -243,11 +326,11 @@ public class LambdaExecutorService {long deadlineMs = System.currentTimeMillis() + (long) fn.getTimeout() * 1000;PendingInvocation invocation = new PendingInvocation(requestId, payload, deadlineMs, fn.getFunctionArn(),- new java.util.concurrent.CompletableFuture<>());+ new CompletableFuture<>());handle.getRuntimeApiServer().enqueue(invocation);- java.util.concurrent.CompletableFuture.anyOf(+ CompletableFuture.anyOf(invocation.getDispatchedFuture(), invocation.getResultFuture()).get(fn.getTimeout() + RUNTIME_DISPATCH_GRACE_SECONDS, TimeUnit.SECONDS);InvokeResult result = invocation.getResultFuture().get();@@ -275,11 +358,56 @@ public class LambdaExecutorService {}}+ /**+ * Drops the pending asynchronous events of a deleted function: AWS runs no further attempt and+ * delivers no record for them. DeleteFunction calls it twice. The call before it drains the warm pool+ * and deletes anything stops the events already queued or running, so none starts, or reports a failure+ * for, a function whose code is being removed. The call after the function has left the store stops an+ * Event invoke that read the count between the two calls and still found the function stored.+ */+ void dropPending(LambdaFunction fn) {+ // ponytail: deleting a single published version does not drop its pending events; they fail+ // to start and are reported as usual, which is rare locally.+ deletions.merge(functionKey(fn), 1L, Long::sum);+ }++ /** Account, region and name, so every version of a function shares one deletion count. */+ private static String functionKey(LambdaFunction fn) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(fn.getFunctionArn());+ return arn.accountId() + ":" + arn.region() + ":" + fn.getFunctionName();+ }++ private boolean stale(AsyncEvent event) {+ return event.generation() != generation.get()+ || event.deletions() != deletions.getOrDefault(functionKey(event.fn()), 0L);+ }+@PreDestroypublic void shutdown() {+ generation.incrementAndGet();asyncExecutor.shutdownNow();}+ /**+ * Drops pending events before storage is wiped, so a retry whose timer fires during the wipe neither+ * runs nor delivers a record. {@link #clear()} moves the generation again for events accepted in between.+ */+ @Override+ public void beforeReset() {+ generation.incrementAndGet();+ }++ @Override+ public void clear() {+ // Only the generation: the deletion counts still fence events from before the reset, so they are+ // kept, one per deleted function, as LambdaService keeps its per-function concurrency locks.+ generation.incrementAndGet();+ }++ private record AsyncEvent(LambdaFunction fn, byte[] payload, String requestId, int chainDepth,+ String invokedQualifier, int maxRetries, long expiresAtMs, long generation,+ long deletions) {}+private byte[] buildErrorPayload(String message, String errorType) {try {ObjectNode node = objectMapper.createObjectNode();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 8e3121e7f5..e8b9de91bc 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -1026,6 +1026,11 @@ public class LambdaService implements ResourceProvider {LambdaFunction fn = getFunction(region, functionName); // throws 404 if not foundfunctionName = fn.getFunctionName();String arn = fn.getFunctionArn();+ // Before the drain and anything is deleted, so no queued or running event starts a container for,+ // or reports on, a function being removed.+ if (executorService != null) {+ executorService.dropPending(fn);+ }warmPool.drainFunction(functionName);// Take the same per-function lock used by Put/DeleteFunctionConcurrency// so a concurrent concurrency mutation cannot interleave with the@@ -1040,6 +1045,10 @@ public class LambdaService implements ResourceProvider {}codeStore.delete(ownerAccount(fn), region, functionName);functionStore.delete(region, functionName);+ // Again now the function is gone, for an Event invoke that read the count after the first call.+ if (executorService != null) {+ executorService.dropPending(fn);+ }reclaimLegacyCodeDirectoryIfUnused(functionName);versionCounters.remove(versionCounterKey(region, fn));versionCounters.remove(legacyVersionCounterKey(region, functionName));@@ -1061,6 +1070,20 @@ public class LambdaService implements ResourceProvider {LOG.infov("Deleted Lambda function: {0}", functionName);}+ /**+ * The {@code $LATEST} record of the function {@code fn} belongs to, whichever version {@code fn} is.+ * Looked up in the account and region of its ARN rather than the caller's.+ */+ Optional<LambdaFunction> findLatest(LambdaFunction fn) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(fn.getFunctionArn());+ return functionStore.getForAccount(arn.accountId(), arn.region(), fn.getFunctionName());+ }++ /** Whether the function {@code fn} belongs to is still in the store, whichever version {@code fn} is. */+ boolean isLive(LambdaFunction fn) {+ return findLatest(fn).isPresent();+ }+/*** Drops the code directory a deleted version owned. Without this the only thing that ever* reclaimed version code was deleting the whole function, so a repeated publish/delete cycle@@ -1105,8 +1128,16 @@ public class LambdaService implements ResourceProvider {}public InvokeResult invoke(String region, String functionName, byte[] payload, InvocationType type) {- LambdaArnUtils.ResolvedFunctionRef ref = LambdaArnUtils.resolve(functionName);- enforceRegion(region, ref);+ return invoke(region, functionName, null, payload, type);+ }++ /**+ * Invokes a function, reconciling a qualifier carried on the name with Invoke's {@code Qualifier} query parameter.+ */+ public InvokeResult invoke(String region, String functionName, String queryQualifier, byte[] payload,+ InvocationType type) {+ validateInvokeQualifier(queryQualifier);+ LambdaArnUtils.ResolvedFunctionRef ref = resolveWithRegion(region, functionName, queryQualifier);String name = ref.name();String qualifier = ref.qualifier();LambdaFunction fn;@@ -2691,6 +2722,13 @@ public class LambdaService implements ResourceProvider {validatePattern(aliasName, "name", ALIAS_NAME_PATTERN);}+ /** Invoke's Qualifier, when supplied: 1-128 characters matching {@link #QUALIFIER}. */+ private static void validateInvokeQualifier(String qualifier) {+ validateNonEmpty(qualifier, "qualifier", false);+ validateMaxLength(qualifier, "qualifier", 128);+ validatePattern(qualifier, "qualifier", QUALIFIER);+ }+public void deleteAlias(String region, String functionName, String aliasName) {validatePattern(aliasName, "name", ALIAS_NAME_PATTERN);String canonical = canonicalFunctionName(region, functionName);diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 58f04c66a0..c9a8e40331 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -363,6 +363,7 @@ floci:code-path: ./data/lambda-codezip-max-entries: 100000poll-interval-ms: 1000+ async-retry-delay-seconds: 60 # FLOCI_SERVICES_LAMBDA_ASYNC_RETRY_DELAY_SECONDS - a failed Event invocation waits this before its first retry and twice this before its second (AWS: 1 min, then 2); 0 retries back to backcontainer-idle-timeout-seconds: 300region-concurrency-limit: 1000unreserved-concurrency-min: 100diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.javaindex 2f4c2522ae..68c7e23106 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaArnInvocationAccountTest.java@@ -14,14 +14,18 @@ import io.github.hectorvent.floci.services.lambda.zip.ZipExtractor;import org.junit.jupiter.api.Test;import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.AdditionalMatchers.aryEq;+import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.isNull;import static org.mockito.ArgumentMatchers.nullable;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -242,6 +246,28 @@ class LambdaArnInvocationAccountTest {assertTrue(rawAliases.get(targetAccount + "/" + aliasKey).isEmpty());}+ @Test+ void deleteFunctionDropsPendingEventsBeforeDrainingAndAfterRemovingTheFunction() {+ String defaultAccount = "000000000000";+ String region = "ap-south-1";+ String functionName = "deleted-function";+ LambdaFunctionStore functionStore = new LambdaFunctionStore(+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, defaultAccount));+ functionStore.save(region, function(functionName,+ "arn:aws:lambda:" + region + ":" + defaultAccount + ":function:" + functionName, "$LATEST"));+ LambdaExecutorService executor = mock(LambdaExecutorService.class);+ WarmPool warmPool = mock(WarmPool.class);+ LambdaService service = service(functionStore, null, executor, warmPool, region, defaultAccount);+ List<String> steps = new ArrayList<>();+ doAnswer(invocation -> steps.add(service.isLive(invocation.getArgument(0)) ? "drop stored" : "drop gone"))+ .when(executor).dropPending(any());+ doAnswer(invocation -> steps.add("drain")).when(warmPool).drainFunction(functionName);++ service.deleteFunction(region, functionName);++ assertEquals(List.of("drop stored", "drain", "drop gone"), steps);+ }+private static LambdaFunction function(String functionName, String functionArn, String version) {LambdaFunction function = new LambdaFunction();function.setFunctionName(functionName);@@ -256,11 +282,21 @@ class LambdaArnInvocationAccountTest {LambdaExecutorService executor,String region,String defaultAccount) {+ return service(functionStore, aliasStore, executor, new WarmPool(), region, defaultAccount);+ }++ private static LambdaService service(+ LambdaFunctionStore functionStore,+ LambdaAliasStore aliasStore,+ LambdaExecutorService executor,+ WarmPool warmPool,+ String region,+ String defaultAccount) {return new LambdaService(functionStore,executor,new LambdaConcurrencyLimiter(),- new WarmPool(),+ warmPool,new CodeStore(Path.of("target/test-data/lambda-code")),new ZipExtractor(),null,diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.javanew file mode 100644index 0000000000..c498d20cbd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java@@ -0,0 +1,226 @@+package io.github.hectorvent.floci.services.lambda;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.Message;+import io.github.hectorvent.floci.services.sqs.model.Queue;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.nio.charset.StandardCharsets;+import java.time.Duration;+import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.zip.ZipEntry;+import java.util.zip.ZipOutputStream;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * Runs a real failing function to prove the asynchronous retries wait as they do on AWS (scaled down by+ * the test config to 1 s then 2 s) and that Invoke's Qualifier selects the alias's asynchronous settings.+ */+@QuarkusTest+class LambdaAsyncRetryDockerIntegrationTest {++ private static final String BASE_PATH = "/2015-03-31";+ private static final String CONFIG_PATH = "/2019-09-25";+ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";+ private static final String ROLE = "arn:aws:iam::" + ACCOUNT + ":role/lambda-role";+ private static final Duration RECEIVE_DEADLINE = Duration.ofSeconds(60);+ private static final ObjectMapper MAPPER = new ObjectMapper();++ @Inject+ DockerClient dockerClient;++ @Inject+ SqsService sqsService;++ @BeforeEach+ void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(),+ "Docker daemon must be available for Lambda asynchronous retry integration tests");+ }++ @Test+ void anAliasInvocationIsRetriedWithGrowingWaitsUnderTheAliasSettings() throws Exception {+ Fixture fixture = setUp();+ try {+ long started = System.nanoTime();+ given()+ .header("X-Amz-Invocation-Type", "Event")+ .body("{\"probe\":\"alias\"}")+ .when()+ .post(BASE_PATH + "/functions/" + fixture.functionName() + "/invocations?Qualifier=live")+ .then()+ .statusCode(202);++ JsonNode record = awaitRecord(fixture.aliasQueueUrl());+ long elapsedMillis = Duration.ofNanos(System.nanoTime() - started).toMillis();++ assertTrue(elapsedMillis >= 3000,+ "the retries should wait 1 s and then 2 s, but the record arrived after " + elapsedMillis + " ms");+ assertEquals("RetriesExhausted", record.path("requestContext").path("condition").asText());+ assertEquals(3, record.path("requestContext").path("approximateInvokeCount").asInt());+ String functionArn = record.path("requestContext").path("functionArn").asText();+ assertTrue(functionArn.endsWith(":1"), "functionArn should name the executed version: " + functionArn);+ assertEquals("1", record.path("responseContext").path("executedVersion").asText());+ assertEquals("alias", record.path("requestPayload").path("probe").asText());+ assertTrue(receiveOnce(fixture.latestQueueUrl()).isEmpty(),+ "the unqualified OnFailure destination should receive nothing");+ } finally {+ deleteFunction(fixture.functionName());+ }+ }++ @Test+ void anUnqualifiedInvocationUsesTheFunctionSettings() throws Exception {+ Fixture fixture = setUp();+ try {+ given()+ .header("X-Amz-Invocation-Type", "Event")+ .body("{\"probe\":\"latest\"}")+ .when()+ .post(BASE_PATH + "/functions/" + fixture.functionName() + "/invocations")+ .then()+ .statusCode(202);++ JsonNode record = awaitRecord(fixture.latestQueueUrl());++ assertEquals("RetriesExhausted", record.path("requestContext").path("condition").asText());+ assertEquals(1, record.path("requestContext").path("approximateInvokeCount").asInt());+ assertEquals("latest", record.path("requestPayload").path("probe").asText());+ assertTrue(receiveOnce(fixture.aliasQueueUrl()).isEmpty(),+ "the alias OnFailure destination should receive nothing");+ } finally {+ deleteFunction(fixture.functionName());+ }+ }++ private Fixture setUp() throws Exception {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String functionName = "async-retry-" + suffix;+ String latestQueueUrl = createQueue("async-retry-latest-" + suffix);+ String aliasQueueUrl = createQueue("async-retry-alias-" + suffix);++ given()+ .contentType("application/json")+ .body("""+ {+ "FunctionName": "%s",+ "Runtime": "nodejs20.x",+ "Role": "%s",+ "Handler": "index.handler",+ "Timeout": 10,+ "Code": {"ZipFile": "%s"}+ }+ """.formatted(functionName, ROLE, failingZipBase64()))+ .when()+ .post(BASE_PATH + "/functions")+ .then()+ .statusCode(201);++ given()+ .contentType("application/json")+ .body("{}")+ .when()+ .post(BASE_PATH + "/functions/" + functionName + "/versions")+ .then()+ .statusCode(201);++ given()+ .contentType("application/json")+ .body("{\"Name\":\"live\",\"FunctionVersion\":\"1\"}")+ .when()+ .post(BASE_PATH + "/functions/" + functionName + "/aliases")+ .then()+ .statusCode(201);++ given()+ .contentType("application/json")+ .body("{\"MaximumRetryAttempts\":0,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"+ .formatted(queueArn(latestQueueUrl)))+ .when()+ .put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config")+ .then()+ .statusCode(200);++ given()+ .contentType("application/json")+ .body("{\"MaximumRetryAttempts\":2,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"+ .formatted(queueArn(aliasQueueUrl)))+ .when()+ .put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config?Qualifier=live")+ .then()+ .statusCode(200);++ return new Fixture(functionName, latestQueueUrl, aliasQueueUrl);+ }++ private String createQueue(String queueName) {+ Queue queue = RequestScopes.callAs(ACCOUNT, () -> sqsService.createQueue(queueName, Map.of(), REGION));+ return queue.getQueueUrl();+ }++ private String queueArn(String queueUrl) {+ return RequestScopes.callAs(ACCOUNT, () ->+ sqsService.getQueueAttributes(queueUrl, List.of("QueueArn"), REGION).get("QueueArn"));+ }++ private List<Message> receiveOnce(String queueUrl) {+ return RequestScopes.callAs(ACCOUNT, () -> sqsService.receiveMessage(queueUrl, 1, 30, 0, REGION));+ }++ private JsonNode awaitRecord(String queueUrl) throws Exception {+ long deadline = System.nanoTime() + RECEIVE_DEADLINE.toNanos();+ while (System.nanoTime() < deadline) {+ List<Message> messages = receiveOnce(queueUrl);+ if (!messages.isEmpty()) {+ return MAPPER.readTree(messages.get(0).getBody());+ }+ Thread.sleep(250);+ }+ return fail("no OnFailure record reached " + queueUrl + " within " + RECEIVE_DEADLINE.toSeconds() + " s");+ }++ private static void deleteFunction(String functionName) {+ given().when().delete(BASE_PATH + "/functions/" + functionName);+ }++ private static String failingZipBase64() throws Exception {+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {+ zip.putNextEntry(new ZipEntry("index.js"));+ zip.write("exports.handler = async () => { throw new Error('boom'); };\n"+ .getBytes(StandardCharsets.UTF_8));+ zip.closeEntry();+ }+ return Base64.getEncoder().encodeToString(bytes.toByteArray());+ }++ private boolean isDockerAvailable() {+ try {+ dockerClient.pingCmd().exec();+ return true;+ } catch (Exception expected) {+ // Docker is unreachable, so requireDocker() skips the test rather than failing it.+ return false;+ }+ }++ private record Fixture(String functionName, String latestQueueUrl, String aliasQueueUrl) {+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.javaindex 4f9d835e9a..dc5ca4ca1c 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorServiceTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.lambda;import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.model.ContainerState;import io.github.hectorvent.floci.services.lambda.model.FunctionEventInvokeConfig;@@ -10,18 +11,24 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;import org.mockito.ArgumentCaptor;+import org.mockito.InOrder;import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;import org.mockito.junit.jupiter.MockitoSettings;import org.mockito.quality.Strictness;+import java.time.Clock;import java.time.Duration;+import java.util.ArrayList;+import java.util.List;import java.util.Optional;import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;@@ -30,12 +37,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyInt;import static org.mockito.ArgumentMatchers.eq;import static org.mockito.ArgumentMatchers.isNull;+import static org.mockito.Mockito.after;import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.inOrder;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.timeout;@@ -50,6 +60,7 @@ class LambdaExecutorServiceTest {@Mock WarmPool warmPool;@Mock LambdaConcurrencyLimiter concurrencyLimiter;+ private final List<LambdaExecutorService> retryExecutors = new ArrayList<>();private LambdaExecutorService executor;private LambdaFunction fn;@@ -65,6 +76,13 @@ class LambdaExecutorServiceTest {when(concurrencyLimiter.acquire(any())).thenReturn(() -> {});}+ @AfterEach+ void shutDownRetryExecutors() {+ for (LambdaExecutorService retryExecutor : retryExecutors) {+ retryExecutor.shutdown();+ }+ }+@Testvoid asyncInvocationPastTheChainBound_isDroppedWithoutRunningTheFunction() {// AWS stops the next invocation in a chain of requests once the same originating event has@@ -78,6 +96,26 @@ class LambdaExecutorServiceTest {verify(warmPool, never()).acquire(any());}+ @Test+ void aVersionTakesItsPermitAgainstTheFunctionsReservation() {+ // A version's snapshot carries no reservation; reserved concurrency is function-wide.+ fn.setReservedConcurrentExecutions(0);+ LambdaFunction version = new LambdaFunction();+ version.setFunctionName("test-fn");+ version.setVersion("1");+ version.setFunctionArn(fn.getFunctionArn() + ":1");+ LambdaService lambdaService = mock(LambdaService.class);+ when(lambdaService.findLatest(version)).thenReturn(Optional.of(fn));+ LambdaExecutorService versionExecutor = new LambdaExecutorService(warmPool, new ObjectMapper(),+ new LambdaConcurrencyLimiter(), null, lambdaService);++ AwsException ex = assertThrows(AwsException.class,+ () -> versionExecutor.invoke(version, "{}".getBytes(), InvocationType.RequestResponse));++ assertEquals("TooManyRequestsException", ex.getErrorCode());+ verify(warmPool, never()).acquire(any());+ }+@Testvoid asyncInvocationShortOfTheChainBound_stillRuns() {executor.invoke(fn, "{}".getBytes(), InvocationType.Event, LambdaInvocationChain.MAX_DEPTH - 1);@@ -282,6 +320,7 @@ class LambdaExecutorServiceTest {config.setMaximumRetryAttempts(2);config.setMaximumEventAgeInSeconds(21600);when(lambdaService.findEventInvokeConfig(fn, null)).thenReturn(Optional.of(config));+ when(lambdaService.isLive(fn)).thenReturn(true);LambdaExecutorService retryExecutor =new LambdaExecutorService(warmPool, new ObjectMapper(), concurrencyLimiter, router, lambdaService);@@ -319,6 +358,7 @@ class LambdaExecutorServiceTest {config.setMaximumRetryAttempts(2);config.setMaximumEventAgeInSeconds(0);when(lambdaService.findEventInvokeConfig(fn, null)).thenReturn(Optional.of(config));+ when(lambdaService.isLive(fn)).thenReturn(true);LambdaExecutorService retryExecutor =new LambdaExecutorService(warmPool, new ObjectMapper(), concurrencyLimiter, router, lambdaService);@@ -348,9 +388,10 @@ class LambdaExecutorServiceTest {config.setMaximumRetryAttempts(2);config.setMaximumEventAgeInSeconds(1);when(lambdaService.findEventInvokeConfig(fn, null)).thenReturn(Optional.of(config));+ when(lambdaService.isLive(fn)).thenReturn(true);MutableClock clock = new MutableClock();- LambdaExecutorService retryExecutor =- new LambdaExecutorService(warmPool, new ObjectMapper(), concurrencyLimiter, router, lambdaService, clock);+ LambdaExecutorService retryExecutor = new LambdaExecutorService(warmPool, new ObjectMapper(),+ concurrencyLimiter, router, lambdaService, clock, Duration.ZERO);RuntimeApiServer rtas = mock(RuntimeApiServer.class);ContainerHandle handle = new ContainerHandle("cid-age-after-failure", "test-fn", rtas, ContainerState.WARM);@@ -381,6 +422,199 @@ class LambdaExecutorServiceTest {"age expiration should preserve the last invocation result");}+ @Test+ void eventInvocation_waitsLongerBeforeEachRetry() throws Exception {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor =+ retryingExecutor(router, eventInvokeConfig(2, 21600), Duration.ofMillis(300));+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ InvokeResult failure = failedAttempt("req-wait");+ List<Long> enqueuedAt = failEveryAttempt(rtas, failure);+ CountDownLatch routed = countRoutes(router);++ byte[] payload = "{}".getBytes();+ retryExecutor.invoke(fn, payload, InvocationType.Event);++ assertTrue(routed.await(10, TimeUnit.SECONDS), "destination routing never ran");+ assertEquals(3, enqueuedAt.size());+ long firstWaitMs = TimeUnit.NANOSECONDS.toMillis(enqueuedAt.get(1) - enqueuedAt.get(0));+ long secondWaitMs = TimeUnit.NANOSECONDS.toMillis(enqueuedAt.get(2) - enqueuedAt.get(1));+ assertTrue(firstWaitMs >= 300, "first retry came after " + firstWaitMs + " ms");+ assertTrue(secondWaitMs >= 600, "second retry came after " + secondWaitMs + " ms");+ verify(router).route(eq(fn), eq(payload), eq(failure), eq(3), eq(0), isNull());+ }++ @Test+ void eventInvocation_releasesTheConcurrencyPermitBetweenAttempts() throws Exception {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor = retryingExecutor(router, eventInvokeConfig(2, 21600), Duration.ZERO);+ LambdaConcurrencyLimiter.Permit permit = mock(LambdaConcurrencyLimiter.Permit.class);+ when(concurrencyLimiter.acquire(fn)).thenReturn(permit);+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ failEveryAttempt(rtas, failedAttempt("req-permit"));+ CountDownLatch routed = countRoutes(router);++ retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ assertTrue(routed.await(10, TimeUnit.SECONDS), "destination routing never ran");+ InOrder inOrder = inOrder(concurrencyLimiter, permit);+ for (int attempt = 1; attempt <= 3; attempt++) {+ inOrder.verify(concurrencyLimiter).acquire(fn);+ inOrder.verify(permit).close();+ }+ inOrder.verifyNoMoreInteractions();+ }++ @Test+ void eventInvocation_throttledRetryPollsAgainWithoutSpendingAnAttempt() throws Exception {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor = retryingExecutor(router, eventInvokeConfig(1, 21600), Duration.ZERO);+ LambdaConcurrencyLimiter.Permit permit = mock(LambdaConcurrencyLimiter.Permit.class);+ when(concurrencyLimiter.acquire(fn))+ .thenReturn(permit)+ .thenThrow(new AwsException("TooManyRequestsException", "Rate exceeded", 429))+ .thenReturn(permit);+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ List<Long> enqueuedAt = failEveryAttempt(rtas, failedAttempt("req-throttled"));+ CountDownLatch routed = countRoutes(router);++ byte[] payload = "{}".getBytes();+ retryExecutor.invoke(fn, payload, InvocationType.Event);++ assertTrue(routed.await(10, TimeUnit.SECONDS), "destination routing never ran");+ assertEquals(2, enqueuedAt.size());+ verify(concurrencyLimiter, times(3)).acquire(fn);+ verify(router).route(eq(fn), eq(payload), any(InvokeResult.class), eq(2), eq(0), isNull());+ }++ @Test+ void eventInvocation_expiryDuringTheRetryWaitRoutesTheLastAttempt() throws Exception {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor =+ retryingExecutor(router, eventInvokeConfig(2, 1), Duration.ofSeconds(10));+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ InvokeResult failure = failedAttempt("req-expiry");+ List<Long> enqueuedAt = failEveryAttempt(rtas, failure);+ CountDownLatch routed = countRoutes(router);++ byte[] payload = "{}".getBytes();+ retryExecutor.invoke(fn, payload, InvocationType.Event);++ assertTrue(routed.await(5, TimeUnit.SECONDS),+ "the retry wait should end when the event expires, not after the full delay");+ assertEquals(1, enqueuedAt.size());+ verify(router).route(eq(fn), eq(payload), eq(failure), eq(1), eq(0), isNull());+ }++ @Test+ void eventInvocation_resetDropsAPendingRetry() {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor =+ retryingExecutor(router, eventInvokeConfig(2, 21600), Duration.ofMillis(500));+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-reset", "test-fn", rtas, ContainerState.WARM);+ when(warmPool.acquire(any())).thenReturn(handle);+ InvokeResult failure = failedAttempt("req-reset");+ doAnswer(invocation -> {+ retryExecutor.clear();+ PendingInvocation pendingInvocation = invocation.getArgument(0);+ pendingInvocation.getResultFuture().complete(failure);+ return pendingInvocation.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ verify(router, after(1500).never()).route(any(), any(), any(), anyInt(), anyInt(), any());+ verify(rtas, times(1)).enqueue(any(PendingInvocation.class));+ }++ @Test+ void eventInvocation_resetDropsAPendingRetryBeforeStorageIsWiped() {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor =+ retryingExecutor(router, eventInvokeConfig(2, 21600), Duration.ofMillis(500));+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-before-reset", "test-fn", rtas, ContainerState.WARM);+ when(warmPool.acquire(any())).thenReturn(handle);+ InvokeResult failure = failedAttempt("req-before-reset");+ doAnswer(invocation -> {+ retryExecutor.beforeReset();+ PendingInvocation pendingInvocation = invocation.getArgument(0);+ pendingInvocation.getResultFuture().complete(failure);+ return pendingInvocation.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ verify(router, after(1500).never()).route(any(), any(), any(), anyInt(), anyInt(), any());+ verify(rtas, times(1)).enqueue(any(PendingInvocation.class));+ }++ @Test+ void eventInvocation_dropsThePendingRetryOfADeletedFunction() {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor = retryingExecutor(router, eventInvokeConfig(2, 21600), Duration.ZERO);+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-deleted", "test-fn", rtas, ContainerState.WARM);+ when(warmPool.acquire(any())).thenReturn(handle);+ InvokeResult failure = failedAttempt("req-deleted");+ doAnswer(invocation -> {+ retryExecutor.dropPending(fn);+ PendingInvocation pendingInvocation = invocation.getArgument(0);+ pendingInvocation.getResultFuture().complete(failure);+ return pendingInvocation.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ verify(router, after(1000).never()).route(any(), any(), any(), anyInt(), anyInt(), any());+ verify(rtas, times(1)).enqueue(any(PendingInvocation.class));+ }++ @Test+ void eventInvocation_dropsTheExpiringEventOfADeletedFunction() {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaExecutorService retryExecutor =+ retryingExecutor(router, eventInvokeConfig(2, 1), Duration.ofSeconds(10));+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ ContainerHandle handle = new ContainerHandle("cid-deleted-expiry", "test-fn", rtas, ContainerState.WARM);+ when(warmPool.acquire(any())).thenReturn(handle);+ InvokeResult failure = failedAttempt("req-deleted-expiry");+ doAnswer(invocation -> {+ retryExecutor.dropPending(fn);+ PendingInvocation pendingInvocation = invocation.getArgument(0);+ pendingInvocation.getResultFuture().complete(failure);+ return pendingInvocation.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));++ retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ verify(router, after(2000).never()).route(any(), any(), any(), anyInt(), anyInt(), any());+ verify(rtas, times(1)).enqueue(any(PendingInvocation.class));+ }++ @Test+ void eventInvocation_ofAFunctionDeletedAfterItWasResolved_isDroppedWithoutAnAttempt() {+ AsyncInvokeDestinationRouter router = mock(AsyncInvokeDestinationRouter.class);+ LambdaService lambdaService = eventInvokeConfig(2, 21600);+ LambdaExecutorService retryExecutor = retryingExecutor(router, lambdaService, Duration.ZERO);+ LambdaConcurrencyLimiter.Permit permit = mock(LambdaConcurrencyLimiter.Permit.class);+ when(concurrencyLimiter.acquire(fn)).thenReturn(permit);+ RuntimeApiServer rtas = mock(RuntimeApiServer.class);+ failEveryAttempt(rtas, failedAttempt("req-deleted-before-invoke"));+ // DeleteFunction ran between the caller resolving fn and this invoke: the function has left+ // the store and its deletion count has already moved.+ when(lambdaService.isLive(fn)).thenReturn(false);+ retryExecutor.dropPending(fn);++ InvokeResult result = retryExecutor.invoke(fn, "{}".getBytes(), InvocationType.Event);++ assertEquals(202, result.getStatusCode());+ verify(warmPool, after(1000).never()).acquire(any());+ verify(router, never()).route(any(), any(), any(), anyInt(), anyInt(), any());+ verify(permit).close();+ }+@Testvoid exceptionDuringInvocation_destroysHandle_doesNotRelease() {RuntimeApiServer rtas = mock(RuntimeApiServer.class);@@ -401,4 +635,49 @@ class LambdaExecutorServiceTest {assertEquals("Unhandled", result.getFunctionError());assertTrue(new String(result.getPayload()).contains("InvocationError"));}++ private LambdaExecutorService retryingExecutor(AsyncInvokeDestinationRouter router, LambdaService lambdaService,+ Duration asyncRetryDelay) {+ LambdaExecutorService retryExecutor = new LambdaExecutorService(warmPool, new ObjectMapper(),+ concurrencyLimiter, router, lambdaService, Clock.systemUTC(), asyncRetryDelay);+ retryExecutors.add(retryExecutor);+ return retryExecutor;+ }++ private LambdaService eventInvokeConfig(int maximumRetryAttempts, int maximumEventAgeInSeconds) {+ FunctionEventInvokeConfig config = new FunctionEventInvokeConfig();+ config.setMaximumRetryAttempts(maximumRetryAttempts);+ config.setMaximumEventAgeInSeconds(maximumEventAgeInSeconds);+ LambdaService lambdaService = mock(LambdaService.class);+ when(lambdaService.findEventInvokeConfig(fn, null)).thenReturn(Optional.of(config));+ when(lambdaService.isLive(fn)).thenReturn(true);+ return lambdaService;+ }++ private static InvokeResult failedAttempt(String requestId) {+ return new InvokeResult(200, "Unhandled", "{\"errorMessage\":\"fails\"}".getBytes(), null, requestId);+ }++ /** Makes every attempt fail with {@code failure}; the list records when each one reached the runtime. */+ private List<Long> failEveryAttempt(RuntimeApiServer rtas, InvokeResult failure) {+ ContainerHandle handle = new ContainerHandle("cid-failing", "test-fn", rtas, ContainerState.WARM);+ when(warmPool.acquire(any())).thenReturn(handle);+ List<Long> enqueuedAt = new CopyOnWriteArrayList<>();+ doAnswer(invocation -> {+ enqueuedAt.add(System.nanoTime());+ PendingInvocation pendingInvocation = invocation.getArgument(0);+ pendingInvocation.getResultFuture().complete(failure);+ return pendingInvocation.getResultFuture();+ }).when(rtas).enqueue(any(PendingInvocation.class));+ return enqueuedAt;+ }++ private static CountDownLatch countRoutes(AsyncInvokeDestinationRouter router) {+ CountDownLatch routed = new CountDownLatch(1);+ doAnswer(invocation -> {+ routed.countDown();+ return null;+ }).when(router).route(any(), any(), any(), anyInt(), anyInt(), any());+ return routed;+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeQualifierIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeQualifierIntegrationTest.javanew file mode 100644index 0000000000..cb07e9cb18--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaInvokeQualifierIntegrationTest.java@@ -0,0 +1,194 @@+package io.github.hectorvent.floci.services.lambda;++import io.quarkus.test.junit.QuarkusTest;+import org.hamcrest.Matcher;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.zip.ZipEntry;+import java.util.zip.ZipOutputStream;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.anyOf;+import static org.hamcrest.Matchers.equalTo;++/**+ * Invoke ignored the {@code Qualifier} query parameter, so {@code ?Qualifier=} ran {@code $LATEST}+ * and an alias's asynchronous invocation settings were never applied.+ */+@QuarkusTest+class LambdaInvokeQualifierIntegrationTest {++ private static String zipB64(String body) throws Exception {+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ try (ZipOutputStream zos = new ZipOutputStream(baos)) {+ zos.putNextEntry(new ZipEntry("handler.py"));+ zos.write(("def handler(e, c):\n return {'body': '" + body + "'}\n")+ .getBytes(StandardCharsets.UTF_8));+ zos.closeEntry();+ }+ return Base64.getEncoder().encodeToString(baos.toByteArray());+ }++ private static Matcher<Integer> anyOf200Or201() {+ return anyOf(equalTo(200), equalTo(201));+ }++ private static String createFunction(String body) throws Exception {+ String name = "inv-qual-" + Long.toString(System.nanoTime(), 36);+ given()+ .contentType("application/json")+ .body("""+ {"FunctionName": "%s", "Runtime": "python3.12",+ "Role": "arn:aws:iam::000000000000:role/r", "Handler": "handler.handler",+ "Code": {"ZipFile": "%s"}}+ """.formatted(name, zipB64(body)))+ .when()+ .post("/2015-03-31/functions")+ .then()+ .statusCode(anyOf200Or201());+ return name;+ }++ private static void publishVersion(String fn, String expectedVersion) {+ given().contentType("application/json").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/versions")+ .then().statusCode(201).body("Version", equalTo(expectedVersion));+ }++ @Test+ void aQualifierQueryParameterSelectsThePublishedVersion() throws Exception {+ String fn = createFunction("v1");+ publishVersion(fn, "1");++ given()+ .contentType("application/json")+ .body("{\"ZipFile\": \"%s\"}".formatted(zipB64("v2")))+ .when()+ .put("/2015-03-31/functions/" + fn + "/code")+ .then()+ .statusCode(200);++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations?Qualifier=1")+ .then().statusCode(204).header("X-Amz-Executed-Version", "1");++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations")+ .then().statusCode(204).header("X-Amz-Executed-Version", "$LATEST");+ }++ @Test+ void aQualifierQueryParameterResolvesAnAlias() throws Exception {+ String fn = createFunction("v1");+ publishVersion(fn, "1");++ given()+ .contentType("application/json")+ .body("{\"Name\": \"live\", \"FunctionVersion\": \"1\"}")+ .when()+ .post("/2015-03-31/functions/" + fn + "/aliases")+ .then()+ .statusCode(201);++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations?Qualifier=live")+ .then().statusCode(204).header("X-Amz-Executed-Version", "1");+ }++ @Test+ void aQualifierQueryParameterThatDisagreesWithTheNameIsRejected() throws Exception {+ String fn = createFunction("v1");+ publishVersion(fn, "1");++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + ":1/invocations?Qualifier=2")+ .then().statusCode(400).body("__type", equalTo("InvalidParameterValueException"));++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + ":1/invocations?Qualifier=1")+ .then().statusCode(204).header("X-Amz-Executed-Version", "1");+ }++ @Test+ void aQualifierQueryParameterThatDoesNotResolveIsNotFound() throws Exception {+ String fn = createFunction("v1");++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations?Qualifier=99")+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ }++ @Test+ void aQualifierQueryParameterAppliesToAnUnqualifiedFunctionArn() throws Exception {+ String fn = createFunction("v1");+ publishVersion(fn, "1");+ String functionArn = given()+ .when().get("/2015-03-31/functions/" + fn + "/configuration")+ .then().statusCode(200).extract().path("FunctionArn");++ given().header("X-Amz-Invocation-Type", "DryRun").body("{}")+ .when().post("/2015-03-31/functions/{functionArn}/invocations?Qualifier=1", functionArn)+ .then().statusCode(204).header("X-Amz-Executed-Version", "1");+ }++ @Test+ void aZeroReservationThrottlesQualifiedInvokes() throws Exception {+ String fn = createFunction("v1");+ publishVersion(fn, "1");+ given()+ .contentType("application/json")+ .body("{\"Name\": \"live\", \"FunctionVersion\": \"1\"}")+ .when()+ .post("/2015-03-31/functions/" + fn + "/aliases")+ .then()+ .statusCode(201);+ given()+ .contentType("application/json")+ .body("{\"ReservedConcurrentExecutions\": 0}")+ .when()+ .put("/2017-10-31/functions/" + fn + "/concurrency")+ .then()+ .statusCode(200);++ for (String target : new String[] {fn + "/invocations?Qualifier=1", fn + "/invocations?Qualifier=live",+ fn + ":1/invocations", fn + ":live/invocations"}) {+ given().header("X-Amz-Invocation-Type", "Event").body("{}")+ .when().post("/2015-03-31/functions/" + target)+ .then().statusCode(429).body("__type", equalTo("TooManyRequestsException"));+ }+ }++ @Test+ void anEmptyQualifierQueryParameterIsRejected() throws Exception {+ String fn = createFunction("v1");++ given().header("X-Amz-Invocation-Type", "DryRun").queryParam("Qualifier", "").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations")+ .then().statusCode(400).body("__type", equalTo("ValidationException"));+ }++ @Test+ void aQualifierQueryParameterLongerThan128CharactersIsRejected() throws Exception {+ String fn = createFunction("v1");++ given().header("X-Amz-Invocation-Type", "DryRun").queryParam("Qualifier", "a".repeat(129)).body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations")+ .then().statusCode(400).body("__type", equalTo("ValidationException"));++ given().header("X-Amz-Invocation-Type", "DryRun").queryParam("Qualifier", "a".repeat(128)).body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations")+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ }++ @Test+ void aQualifierQueryParameterWithAnInvalidCharacterIsRejected() throws Exception {+ String fn = createFunction("v1");++ given().header("X-Amz-Invocation-Type", "DryRun").queryParam("Qualifier", " ").body("{}")+ .when().post("/2015-03-31/functions/" + fn + "/invocations")+ .then().statusCode(400).body("__type", equalTo("ValidationException"));+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 5280659310..aae5bcd737 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -150,6 +150,7 @@ floci:docker-flags: ""honour-architectures: falsezip-max-entries: 100000+ async-retry-delay-seconds: 1# aws-config-path:hot-reload:enabled: true