Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): honor MaximumBatchingWindowInSeconds on SQS event source mappings (#3203)
Vũ Thế Huyec2d678e6d90
diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex 998c99256c..f165fb41ee 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -737,6 +737,29 @@ aws lambda create-event-source-mapping \--endpoint-url $AWS_ENDPOINT_URL```+### MaximumBatchingWindowInSeconds (SQS)++`CreateEventSourceMapping` and `UpdateEventSourceMapping` accept a+`MaximumBatchingWindowInSeconds` integer between 0 and 300. `GetEventSourceMapping`+and `ListEventSourceMappings` echo it back when set; responses omit the field when+it was never configured. Values outside 0 to 300 are rejected with+`InvalidParameterValueException`.++When the window is greater than 0, the SQS poller holds an underfilled batch open,+accumulating messages across polls, and invokes the function once the batch reaches+`BatchSize` or the window elapses since the first buffered message, whichever comes+first. A window of 0 (or an unset window) invokes as soon as any message is+available, which is the previous behaviour.++```bash+aws lambda create-event-source-mapping \+ --function-name my-function \+ --event-source-arn $QUEUE_ARN \+ --batch-size 10 \+ --maximum-batching-window-in-seconds 5 \+ --endpoint-url $AWS_ENDPOINT_URL+```+### ScalingConfig (SQS only)`CreateEventSourceMapping` and `UpdateEventSourceMapping` accept adiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.javaindex 9a06dbdd8f..ca7ff325c9 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java@@ -349,6 +349,9 @@ public class LambdaController {node.put("EventSourceArn", esm.getEventSourceArn());}node.put("BatchSize", esm.getBatchSize());+ if (esm.getMaximumBatchingWindowInSeconds() != null) {+ node.put("MaximumBatchingWindowInSeconds", esm.getMaximumBatchingWindowInSeconds());+ }node.put("State", esm.getState());node.put("LastModified", (double) esm.getLastModified() / 1000.0);// Omitted rather than nulled when unset, so a mapping created without a starting positiondiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 08eb61e83a..0512207290 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -1236,6 +1236,7 @@ public class LambdaService implements ResourceProvider {ResolvedFunctionTarget target = resolveFunctionTarget(resolvedRegion, fnRef);int batchSize = toInt(request.get("BatchSize"), 10);+ Integer maximumBatchingWindowInSeconds = parseMaximumBatchingWindow(request);boolean enabled = !Boolean.FALSE.equals(request.get("Enabled"));@SuppressWarnings("unchecked")@@ -1268,6 +1269,7 @@ public class LambdaService implements ResourceProvider {esm.setQueueUrl(queueUrl);esm.setRegion(resolvedRegion);esm.setBatchSize(batchSize);+ esm.setMaximumBatchingWindowInSeconds(maximumBatchingWindowInSeconds);esm.setEnabled(enabled);esm.setState(enabled ? "Enabled" : "Disabled");esm.setScalingConfig(scalingConfig);@@ -1629,6 +1631,33 @@ public class LambdaService implements ResourceProvider {return new ScalingConfig((int) longValue);}+ /**+ * Parses {@code MaximumBatchingWindowInSeconds} out of a create/update request and applies+ * AWS-level validation: it must be an integer in [0, 300]. Returns {@code null} when the field+ * is absent so a create leaves it unset and an update leaves the stored value untouched.+ */+ private Integer parseMaximumBatchingWindow(Map<String, Object> request) {+ Object raw = request.get("MaximumBatchingWindowInSeconds");+ if (raw == null) {+ return null;+ }+ if (!(raw instanceof Number)) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumBatchingWindowInSeconds must be a numeric value", 400);+ }+ double d = ((Number) raw).doubleValue();+ if (Double.isNaN(d) || Double.isInfinite(d) || d != Math.floor(d)) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumBatchingWindowInSeconds must be an integer", 400);+ }+ long value = ((Number) raw).longValue();+ if (value < 0 || value > 300) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumBatchingWindowInSeconds must be between 0 and 300 (got " + value + ")", 400);+ }+ return (int) value;+ }+private void startPollingHelper(EventSourceMapping esm) {if (esm.getEventSourceArn() == null) {return;@@ -1681,6 +1710,9 @@ public class LambdaService implements ResourceProvider {if (request.containsKey("BatchSize")) {esm.setBatchSize(toInt(request.get("BatchSize"), esm.getBatchSize()));}+ if (request.containsKey("MaximumBatchingWindowInSeconds")) {+ esm.setMaximumBatchingWindowInSeconds(parseMaximumBatchingWindow(request));+ }if (request.containsKey("Enabled")) {boolean nowEnabled = !Boolean.FALSE.equals(request.get("Enabled"));esm.setEnabled(nowEnabled);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.javaindex dd27622d0d..be1c95a6ed 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java@@ -29,6 +29,7 @@ import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.function.LongSupplier;import java.util.stream.Collectors;/**@@ -57,6 +58,11 @@ public class SqsEventSourcePoller implements Resettable {private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();// Tracks ESMs with an in-flight poll to prevent concurrent deliveries of the same messageprivate final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();+ // Underfilled batches held open by MaximumBatchingWindowInSeconds, keyed by ESM uuid. Only ever+ // mutated inside the activePolls-guarded section, so one thread touches a given entry at a time.+ private final ConcurrentHashMap<String, PendingBatch> pendingBatches = new ConcurrentHashMap<>();+ // Wall clock for the batching-window deadline; overridable so tests can advance it without sleeping.+ private volatile LongSupplier clockMs = System::currentTimeMillis;private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {Thread t = new Thread(r, "esm-poller");t.setDaemon(true);@@ -103,6 +109,18 @@ public class SqsEventSourcePoller implements Resettable {timerIds.values().forEach(vertx::cancelTimer);timerIds.clear();activePolls.clear();+ pendingBatches.clear();+ }++ /** Test seam: replace the wall clock used for the batching-window deadline. */+ void setClockForTest(LongSupplier clockMs) {+ this.clockMs = clockMs;+ }++ /** Messages received across earlier polls that a batching window is still holding open. */+ private static final class PendingBatch {+ final List<Message> messages = new ArrayList<>();+ long windowStartedAtMs;}public void startPolling(EventSourceMapping esm) {@@ -127,6 +145,7 @@ public class SqsEventSourcePoller implements Resettable {public void stopPolling(String uuid) {Long timerId = timerIds.remove(uuid);+ pendingBatches.remove(uuid);if (timerId != null) {vertx.cancelTimer(timerId);LOG.debugv("Stopped polling ESM {0}", uuid);@@ -154,12 +173,56 @@ public class SqsEventSourcePoller implements Resettable {return;}- int visibilityTimeout = fn.getTimeout() + 30;- List<Message> messages = sqsService.receiveMessage(- esm.getQueueUrl(), esm.getBatchSize(), visibilityTimeout, 0, esm.getRegion());+ // MaximumBatchingWindowInSeconds holds an underfilled batch open: keep messages+ // received across earlier polls invisible and only invoke once the batch fills or+ // the window since the first buffered message expires.+ int window = esm.getMaximumBatchingWindowInSeconds() == null+ ? 0 : esm.getMaximumBatchingWindowInSeconds();+ // A message can be buffered for the whole window and then processed for the full+ // function timeout, so visibility has to cover both plus AWS's 30s margin, not the+ // larger of the two. Otherwise a long invoke can outlive the visibility and another+ // consumer receives the message, causing a duplicate delivery and a stale handle.+ int visibilityTimeout = fn.getTimeout() + Math.max(window, 0) + 30;- if (messages.isEmpty()) {- return;+ PendingBatch pending = pendingBatches.get(esm.getUuid());+ int alreadyBuffered = pending != null ? pending.messages.size() : 0;+ int wanted = Math.max(1, esm.getBatchSize() - alreadyBuffered);++ List<Message> received = sqsService.receiveMessage(+ esm.getQueueUrl(), wanted, visibilityTimeout, 0, esm.getRegion());++ List<Message> messages;+ if (window <= 0) {+ // No window, or one that was just turned off: deliver now, draining anything a+ // previous positive window had buffered so those messages are not stranded+ // until their visibility expires (and cannot be replayed if the window returns).+ List<Message> batch = pending != null ? pending.messages : new ArrayList<>();+ pendingBatches.remove(esm.getUuid());+ batch.addAll(received);+ if (batch.isEmpty()) {+ return;+ }+ messages = batch;+ } else {+ if (pending == null) {+ pending = new PendingBatch();+ pendingBatches.put(esm.getUuid(), pending);+ }+ if (pending.messages.isEmpty() && !received.isEmpty()) {+ pending.windowStartedAtMs = clockMs.getAsLong();+ }+ pending.messages.addAll(received);+ if (pending.messages.isEmpty()) {+ return;+ }+ boolean batchFull = pending.messages.size() >= esm.getBatchSize();+ boolean windowElapsed =+ clockMs.getAsLong() - pending.windowStartedAtMs >= window * 1000L;+ if (!batchFull && !windowElapsed) {+ return;+ }+ messages = new ArrayList<>(pending.messages);+ pendingBatches.remove(esm.getUuid());}LOG.infov("ESM {0}: received {1} message(s)", esm.getUuid(), messages.size());diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.javaindex 915ffc6512..00e9052031 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java@@ -22,6 +22,7 @@ public class EventSourceMapping {private String region;private boolean enabled = true;private int batchSize = 10;+ private Integer maximumBatchingWindowInSeconds;private String state = "Enabled";private long lastModified;private String startingPosition;@@ -66,6 +67,12 @@ public class EventSourceMapping {public int getBatchSize() { return batchSize; }public void setBatchSize(int batchSize) { this.batchSize = batchSize; }+ /** Seconds to accumulate an underfilled batch before invoking; {@code null} or 0 means invoke as soon as messages arrive. */+ public Integer getMaximumBatchingWindowInSeconds() { return maximumBatchingWindowInSeconds; }+ public void setMaximumBatchingWindowInSeconds(Integer maximumBatchingWindowInSeconds) {+ this.maximumBatchingWindowInSeconds = maximumBatchingWindowInSeconds;+ }+public String getState() { return state; }public void setState(String state) { this.state = state; }diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.javaindex 37030bdf08..51192d0064 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java@@ -140,6 +140,62 @@ class EsmIntegrationTest {given().delete(LAMBDA_BASE + "/event-source-mappings/" + uuid).then().statusCode(202);}+ @Test+ void maximumBatchingWindowRoundTripsThroughCreateGetAndUpdate() {+ String uuid = given()+ .contentType("application/json")+ .body("""+ {+ "FunctionName": "%s",+ "EventSourceArn": "%s",+ "BatchSize": 2,+ "MaximumBatchingWindowInSeconds": 5+ }+ """.formatted(FUNCTION_NAME, QUEUE_ARN))+ .when()+ .post(LAMBDA_BASE + "/event-source-mappings")+ .then()+ .statusCode(202)+ .body("MaximumBatchingWindowInSeconds", equalTo(5))+ .extract()+ .path("UUID");++ given()+ .when()+ .get(LAMBDA_BASE + "/event-source-mappings/" + uuid)+ .then()+ .statusCode(200)+ .body("MaximumBatchingWindowInSeconds", equalTo(5));++ given()+ .contentType("application/json")+ .body("{\"MaximumBatchingWindowInSeconds\": 0}")+ .when()+ .put(LAMBDA_BASE + "/event-source-mappings/" + uuid)+ .then()+ .statusCode(202)+ .body("MaximumBatchingWindowInSeconds", equalTo(0));++ given().delete(LAMBDA_BASE + "/event-source-mappings/" + uuid).then().statusCode(202);+ }++ @Test+ void createEventSourceMappingRejectsMaximumBatchingWindowAbove300() {+ given()+ .contentType("application/json")+ .body("""+ {+ "FunctionName": "%s",+ "EventSourceArn": "%s",+ "MaximumBatchingWindowInSeconds": 301+ }+ """.formatted(FUNCTION_NAME, QUEUE_ARN))+ .when()+ .post(LAMBDA_BASE + "/event-source-mappings")+ .then()+ .statusCode(400);+ }+@Testvoid updateEventSourceMappingReturnsFailureConfig() {String streamArn = "arn:aws:dynamodb:us-east-1:000000000000:table/esm-table/stream/2026-01-01T00:00:00.000";diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EventSourceMappingSerializationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EventSourceMappingSerializationTest.javaindex 0e18bc2a6d..d1925b323f 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EventSourceMappingSerializationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EventSourceMappingSerializationTest.java@@ -96,6 +96,38 @@ class EventSourceMappingSerializationTest {assertNotNull(response.get("SourceAccessConfigurations"));}+ @Test+ void maximumBatchingWindowRoundTripsThroughJacksonAndIsOmittedWhenUnset() throws Exception {+ EventSourceMapping esm = new EventSourceMapping();+ esm.setUuid("esm-window-001");+ esm.setMaximumBatchingWindowInSeconds(5);++ EventSourceMapping deserialized = objectMapper.readValue(+ objectMapper.writeValueAsString(esm), EventSourceMapping.class);+ assertEquals(5, deserialized.getMaximumBatchingWindowInSeconds());++ assertNull(new EventSourceMapping().getMaximumBatchingWindowInSeconds());+ }++ @Test+ void buildEsmResponseEmitsMaximumBatchingWindowOnlyWhenSet() {+ EventSourceMapping esm = new EventSourceMapping();+ esm.setUuid("esm-window-002");+ esm.setFunctionArn("arn:aws:lambda:us-east-1:123456789012:function:sqsConsumer");+ esm.setEventSourceArn("arn:aws:sqs:us-east-1:123456789012:my-queue");+ esm.setBatchSize(2);+ esm.setState("Enabled");+ esm.setLastModified(1700000000000L);++ assertFalse(controller.buildEsmResponse(esm).containsKey("MaximumBatchingWindowInSeconds"));++ esm.setMaximumBatchingWindowInSeconds(5);+ assertEquals(5, controller.buildEsmResponse(esm).get("MaximumBatchingWindowInSeconds"));++ esm.setMaximumBatchingWindowInSeconds(0);+ assertEquals(0, controller.buildEsmResponse(esm).get("MaximumBatchingWindowInSeconds"));+ }+@Testvoid testBuildEsmResponseWithEventSourceArnPresent() {EventSourceMapping esm = new EventSourceMapping();diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.javaindex 4ddfc5f99b..bfbe578d9d 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePollerTest.java@@ -21,6 +21,7 @@ import java.time.Instant;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.atomic.AtomicLong;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;@@ -473,6 +474,134 @@ class SqsEventSourcePollerTest {}}+ private static void sleep(long millis) {+ try {+ Thread.sleep(millis);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ }+ }++ private void pollUntilInvoked(EventSourceMapping esm, LambdaFunction fn) {+ long deadline = System.currentTimeMillis() + 5000;+ while (true) {+ poller.pollAndInvoke(esm);+ try {+ verify(executorService, timeout(2000))+ .invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse));+ return;+ } catch (AssertionError retry) {+ if (System.currentTimeMillis() > deadline) {+ throw retry;+ }+ sleep(25);+ }+ }+ }++ @Test+ void batchingWindowHoldsUnderfilledBatchUntilTheWindowElapses() {+ LambdaFunction fn = stubThrowFn();+ EventSourceMapping esm = esm();+ esm.setBatchSize(2);+ esm.setMaximumBatchingWindowInSeconds(5);++ AtomicLong now = new AtomicLong(0);+ poller.setClockForTest(now::get);++ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(message("m1")))+ .thenReturn(List.of());+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());++ // Polls inside the window buffer the record without invoking.+ for (int i = 0; i < 4; i++) {+ poller.pollAndInvoke(esm);+ sleep(25);+ now.addAndGet(1000); // reaches 4s, short of the 5s window+ }+ verify(sqsService, atLeast(1))+ .receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1"));+ verify(executorService, never()).invoke(any(), any(), any());+ verify(sqsService, never()).deleteMessage(any(), any(), any());++ // Once the window elapses the buffered record is delivered.+ now.set(5000);+ pollUntilInvoked(esm, fn);+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");+ }++ @Test+ void batchingWindowFlushesEarlyOnceTheBatchIsFull() {+ LambdaFunction fn = stubThrowFn();+ EventSourceMapping esm = esm();+ esm.setBatchSize(2);+ esm.setMaximumBatchingWindowInSeconds(30);++ poller.setClockForTest(() -> 0L); // the window never expires on its own++ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(message("m1")))+ .thenReturn(List.of(message("m2")))+ .thenReturn(List.of());+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());++ pollUntilInvoked(esm, fn);++ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m2", "us-east-1");+ }++ @Test+ void batchingWindowVisibilityTimeoutCoversTheWindowPlusFunctionTimeout() {+ EventSourceMapping esm = esm();+ esm.setMaximumBatchingWindowInSeconds(60);+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("throwfn");+ fn.setTimeout(120);+ when(functionStore.getForAccount("000000000000", "us-east-1", "throwfn"))+ .thenReturn(Optional.of(fn));+ poller.setClockForTest(() -> 0L);+ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of());++ poller.pollAndInvoke(esm);++ ArgumentCaptor<Integer> visibility = ArgumentCaptor.forClass(Integer.class);+ verify(sqsService, timeout(2000)).receiveMessage(+ eq(esm.getQueueUrl()), anyInt(), visibility.capture(), anyInt(), eq("us-east-1"));+ // held up to 60s buffered, then up to 120s executing, plus AWS's 30s margin+ assertEquals(120 + 60 + 30, visibility.getValue());+ }++ @Test+ void turningTheBatchingWindowOffDeliversAnAlreadyBufferedBatch() {+ LambdaFunction fn = stubThrowFn();+ EventSourceMapping esm = esm();+ esm.setBatchSize(10);+ esm.setMaximumBatchingWindowInSeconds(300);+ poller.setClockForTest(() -> 0L); // the window never elapses on its own++ when(sqsService.receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1")))+ .thenReturn(List.of(message("m1")))+ .thenReturn(List.of());+ when(executorService.invoke(eq(fn), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());++ // Poll once with the window open: the record is buffered, not delivered.+ poller.pollAndInvoke(esm);+ verify(sqsService, timeout(2000))+ .receiveMessage(eq(esm.getQueueUrl()), anyInt(), anyInt(), anyInt(), eq("us-east-1"));+ verify(executorService, never()).invoke(any(), any(), any());++ // Window turned off: the next poll must flush the stranded batch instead of ignoring it.+ esm.setMaximumBatchingWindowInSeconds(0);+ pollUntilInvoked(esm, fn);+ verify(sqsService, timeout(2000)).deleteMessage(esm.getQueueUrl(), "rh-m1", "us-east-1");+ }+private static final String BODY_PATTERN = "{\"body\":{\"type\":[\"order\"]}}";@Test