Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): back off DynamoDB Streams retries and expire aged records (#3613)
Chris Jovanovski86072aaa8438
diff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex 92fab8888c..c36705bdf6 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -753,6 +753,8 @@ aws lambda update-function-code \Connect Lambda to SQS, Kinesis, or DynamoDB Streams. Self-managed Apache Kafka event source mappings are accepted, validated, persisted, and returned on the wire, but Floci does not run an active Kafka consumer poller:+For DynamoDB Streams mappings, Floci retries failed batches with exponential backoff, honors `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds`, and sends discarded batches to configured SQS or SNS `DestinationConfig.OnFailure` destinations.+```bash# SQS triggerQUEUE_ARN=$(aws sqs get-queue-attributes \diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex d5d1177da4..0f2afc64a6 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -39,6 +39,7 @@ import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.function.LongSupplier;@ApplicationScopedpublic class DynamoDbStreamsEventSourcePoller implements Resettable {@@ -51,6 +52,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {/** Raised by the stream when a stored checkpoint has aged out of the retained window. */private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";+ static final long MAX_RETRY_BACKOFF_MS = 60_000;private final Vertx vertx;private final DynamoDbStreamService streamService;@@ -64,9 +66,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private final S3Service s3Service;private final String baseUrl;private final long pollIntervalMs;+ private final LongSupplier clock;private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Integer> retryCounts = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {Thread t = new Thread(r, "dynamodb-streams-esm-poller");t.setDaemon(true);@@ -84,6 +88,21 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {SqsService sqsService,SnsService snsService,S3Service s3Service) {+ this(vertx, streamService, executorService, functionStore, esmStore, objectMapper, config,+ filterMatcher, sqsService, snsService, s3Service, System::currentTimeMillis);+ }++ DynamoDbStreamsEventSourcePoller(Vertx vertx, DynamoDbStreamService streamService,+ LambdaExecutorService executorService,+ LambdaFunctionStore functionStore,+ EsmStore esmStore,+ ObjectMapper objectMapper,+ EmulatorConfig config,+ PipesFilterMatcher filterMatcher,+ SqsService sqsService,+ SnsService snsService,+ S3Service s3Service,+ LongSupplier clock) {this.vertx = vertx;this.streamService = streamService;this.executorService = executorService;@@ -96,6 +115,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {this.sqsService = sqsService;this.snsService = snsService;this.s3Service = s3Service;+ this.clock = clock;}public void startPersistedPollers() {@@ -145,6 +165,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {timerIds.clear();activePolls.clear();retryCounts.clear();+ retryNotBefore.clear();}public void startPolling(EventSourceMapping esm) {@@ -170,6 +191,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {LOG.debugv("Stopped DynamoDB Streams polling for ESM {0}", uuid);}retryCounts.keySet().removeIf(k -> k.startsWith(uuid + ":"));+ retryNotBefore.keySet().removeIf(k -> k.startsWith(uuid + ":"));}@@ -241,6 +263,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {return;}+ String checkpointSeq = (lastSeq == null || checkpointTrimmed) ? "TRIM_HORIZON" : lastSeq;+ String batchKey = esm.getUuid() + ":" + shardId + ":" + checkpointSeq;+ long now = clock.getAsLong();+ if (now < retryNotBefore.getOrDefault(batchKey, 0L)) {+ return;+ }+LOG.infov("DynamoDB Streams ESM {0}: delivering {1} of {2} record(s) to {3}",esm.getUuid(), matched.size(), records.size(), esm.getFunctionName());@@ -257,14 +286,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {throw e;}- String checkpointSeq = (lastSeq == null || checkpointTrimmed) ? "TRIM_HORIZON" : lastSeq;- String batchKey = esm.getUuid() + ":" + shardId + ":" + checkpointSeq;CheckpointOutcome outcome = invokeResult.getFunctionError() == null? successfulInvocationCheckpoint(esm, invokeResult, lastSeq, records, matched): new CheckpointOutcome(null, 0);if (outcome.lowestFailedIndex() == records.size()) {retryCounts.remove(batchKey);+ retryNotBefore.remove(batchKey);advanceCheckpoint(esm, shardId, outcome.checkpoint());} else if (outcome.lowestFailedIndex() > 0) {retryCounts.remove(batchKey);@@ -273,28 +301,39 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {String nextBatchKey = esm.getUuid() + ":" + shardId + ":" + nextCheckpoint;Integer maxRetries = esm.getMaximumRetryAttempts();int currentRetries = retryCounts.merge(nextBatchKey, 1, Integer::sum);+ Set<String> deliveredSeqs = new HashSet<>();+ for (DynamoDbStreamRecord rec : matched) {+ deliveredSeqs.add(rec.getSequenceNumber());+ }+ List<DynamoDbStreamRecord> failedRecords = new ArrayList<>();+ for (int i = outcome.lowestFailedIndex(); i < records.size(); i++) {+ DynamoDbStreamRecord rec = records.get(i);+ if (deliveredSeqs.contains(rec.getSequenceNumber())) {+ failedRecords.add(rec);+ }+ }+ if (failedRecords.isEmpty()) {+ failedRecords = records.subList(outcome.lowestFailedIndex(), records.size());+ }- if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {+ if (hasExceededMaximumRecordAge(esm, failedRecords, now)) {+ LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",+ esm.getUuid(), newestFetchedSeq);+ sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,+ "MaximumRecordAgeExceeded");+ retryCounts.remove(nextBatchKey);+ retryNotBefore.remove(nextBatchKey);+ advanceCheckpoint(esm, shardId, newestFetchedSeq);+ } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",esm.getUuid(), maxRetries, newestFetchedSeq);- Set<String> deliveredSeqs = new HashSet<>();- for (DynamoDbStreamRecord rec : matched) {- deliveredSeqs.add(rec.getSequenceNumber());- }- List<DynamoDbStreamRecord> failedRecords = new ArrayList<>();- for (int i = outcome.lowestFailedIndex(); i < records.size(); i++) {- DynamoDbStreamRecord rec = records.get(i);- if (deliveredSeqs.contains(rec.getSequenceNumber())) {- failedRecords.add(rec);- }- }- if (failedRecords.isEmpty()) {- failedRecords = records.subList(outcome.lowestFailedIndex(), records.size());- }- sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries);+ sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,+ "RetryAttemptsExhausted");retryCounts.remove(nextBatchKey);+ retryNotBefore.remove(nextBatchKey);advanceCheckpoint(esm, shardId, newestFetchedSeq);} else {+ retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",esm.getUuid(), currentRetries);advanceCheckpoint(esm, shardId, nextCheckpoint);@@ -302,13 +341,24 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {} else {Integer maxRetries = esm.getMaximumRetryAttempts();int currentRetries = retryCounts.merge(batchKey, 1, Integer::sum);- if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {+ if (hasExceededMaximumRecordAge(esm, matched, now)) {+ LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",+ esm.getUuid(), newestFetchedSeq);+ sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,+ "MaximumRecordAgeExceeded");+ retryCounts.remove(batchKey);+ retryNotBefore.remove(batchKey);+ advanceCheckpoint(esm, shardId, newestFetchedSeq);+ } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",esm.getUuid(), maxRetries, newestFetchedSeq);- sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries);+ sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,+ "RetryAttemptsExhausted");retryCounts.remove(batchKey);+ retryNotBefore.remove(batchKey);advanceCheckpoint(esm, shardId, newestFetchedSeq);} else {+ retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));String error = invokeResult.getFunctionError() != null? invokeResult.getFunctionError(): "batchItemFailures";@@ -399,10 +449,34 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {return previousCheckpoint;}+ private boolean hasExceededMaximumRecordAge(EventSourceMapping esm, List<DynamoDbStreamRecord> records,+ long now) {+ Integer configuredAge = esm.getMaximumRecordAgeInSeconds();+ if (configuredAge == null || configuredAge <= 0) {+ return false;+ }++ long oldestRecordSeconds = records.stream()+ .mapToLong(DynamoDbStreamRecord::getApproximateCreationDateTime)+ .filter(seconds -> seconds > 0)+ .min()+ .orElse(0);+ if (oldestRecordSeconds == 0) {+ return false;+ }+ return now - oldestRecordSeconds * 1_000 >= configuredAge * 1_000L;+ }++ long retryBackoffMs(int retries) {+ int doublings = Math.min(retries, 16);+ return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);+ }+private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,List<DynamoDbStreamRecord> records,InvokeResult invokeResult,- int invokeCount) {+ int invokeCount,+ String condition) {if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {return;}@@ -415,12 +489,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {if (destinationArn.contains(":sqs:")) {String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);sqsService.sendMessage(queueUrl, payload, 0, region);LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SQS DLQ {1}", esm.getUuid(), destinationArn);} else if (destinationArn.contains(":sns:")) {String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);snsService.publish(destinationArn, null, payload, "ESM OnFailure", region);LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SNS DLQ {1}", esm.getUuid(), destinationArn);} else if (destinationArn.contains(":s3:")) {@@ -429,7 +503,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);}String key = buildS3OnFailureKey(esm.getUuid(), shardId, Instant.now(), UUID.randomUUID());- String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount);+ String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);RequestScopes.runAs(esm.getAccountId(), () ->s3Service.putObject(arn.resource(), key,s3Payload.getBytes(StandardCharsets.UTF_8),@@ -449,10 +523,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,List<DynamoDbStreamRecord> records,InvokeResult invokeResult,- int invokeCount) {+ int invokeCount,+ String condition) {try {ObjectNode root = (ObjectNode) objectMapper.readTree(- buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount));+ buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition));root.put("payload", buildDynamoDbEvent(records, esm));return objectMapper.writeValueAsString(root);} catch (Exception e) {@@ -469,7 +544,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private String buildOnFailurePayload(EventSourceMapping esm, String shardId,List<DynamoDbStreamRecord> records,InvokeResult invokeResult,- int invokeCount) {+ int invokeCount,+ String condition) {try {ObjectNode root = objectMapper.createObjectNode();root.put("version", "1.0");@@ -478,7 +554,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {ObjectNode requestContext = root.putObject("requestContext");requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");- requestContext.put("condition", "RetryAttemptsExhausted");+ requestContext.put("condition", condition);requestContext.put("approximateInvokeCount", invokeCount);ObjectNode responseContext = root.putObject("responseContext");diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.javaindex b2cf08c25f..50b12dfd19 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java@@ -372,6 +372,9 @@ public class LambdaController {if (esm.getMaximumRetryAttempts() != null) {node.put("MaximumRetryAttempts", esm.getMaximumRetryAttempts());}+ if (esm.getMaximumRecordAgeInSeconds() != null) {+ node.put("MaximumRecordAgeInSeconds", esm.getMaximumRecordAgeInSeconds());+ }if (esm.getDestinationConfig() != null && esm.getDestinationConfig().getOnFailure() != null) {ObjectNode destinationConfig = node.putObject("DestinationConfig");diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 5ca0725cdc..a92d8a5885 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -1320,6 +1320,7 @@ public class LambdaService implements ResourceProvider {: null;Integer maximumRetryAttempts = parseMaximumRetryAttempts(request);+ Integer maximumRecordAgeInSeconds = parseMaximumRecordAgeInSeconds(request);EventSourceMapping.DestinationConfig destinationConfig = parseDestinationConfig(request);@@ -1347,6 +1348,7 @@ public class LambdaService implements ResourceProvider {esm.setFunctionResponseTypes(functionResponseTypes);esm.setBisectBatchOnFunctionError(bisectBatchOnFunctionError);esm.setMaximumRetryAttempts(maximumRetryAttempts);+ esm.setMaximumRecordAgeInSeconds(maximumRecordAgeInSeconds);esm.setDestinationConfig(destinationConfig);esm.setFilterCriteria(filterCriteria);esm.setStartingPosition(startingPosition.position());@@ -1752,6 +1754,28 @@ public class LambdaService implements ResourceProvider {return (int) value;}+ private Integer parseMaximumRecordAgeInSeconds(Map<String, Object> request) {+ Object raw = request.get("MaximumRecordAgeInSeconds");+ if (raw == null) {+ return null;+ }+ if (!(raw instanceof Number)) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumRecordAgeInSeconds must be a numeric value", 400);+ }+ double d = ((Number) raw).doubleValue();+ if (Double.isNaN(d) || Double.isInfinite(d) || d != Math.floor(d)) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumRecordAgeInSeconds must be an integer", 400);+ }+ long value = ((Number) raw).longValue();+ if (value != -1 && (value < 60 || value > 604800)) {+ throw new AwsException("InvalidParameterValueException",+ "MaximumRecordAgeInSeconds must be -1 or between 60 and 604800 (got " + value + ")", 400);+ }+ return (int) value;+ }+private void startPollingHelper(EventSourceMapping esm) {if (esm.getEventSourceArn() == null) {return;@@ -1826,6 +1850,9 @@ public class LambdaService implements ResourceProvider {if (request.containsKey("MaximumRetryAttempts")) {esm.setMaximumRetryAttempts(parseMaximumRetryAttempts(request));}+ if (request.containsKey("MaximumRecordAgeInSeconds")) {+ esm.setMaximumRecordAgeInSeconds(parseMaximumRecordAgeInSeconds(request));+ }if (request.containsKey("DestinationConfig")) {esm.setDestinationConfig(parseDestinationConfig(request));diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.javaindex 32e999cf92..b3662043b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java@@ -32,6 +32,7 @@ public class EventSourceMapping {private ScalingConfig scalingConfig;private Boolean bisectBatchOnFunctionError;private Integer maximumRetryAttempts;+ private Integer maximumRecordAgeInSeconds;private DestinationConfig destinationConfig;private FilterCriteria filterCriteria;private Map<String, Object> selfManagedEventSource;@@ -127,6 +128,14 @@ public class EventSourceMapping {this.maximumRetryAttempts = maximumRetryAttempts;}+ public Integer getMaximumRecordAgeInSeconds() {+ return maximumRecordAgeInSeconds;+ }++ public void setMaximumRecordAgeInSeconds(Integer maximumRecordAgeInSeconds) {+ this.maximumRecordAgeInSeconds = maximumRecordAgeInSeconds;+ }+public DestinationConfig getDestinationConfig() {return destinationConfig;}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex dad0016048..52d9d61fbd 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -42,6 +42,7 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.UUID;+import java.util.concurrent.atomic.AtomicLong;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -86,12 +87,14 @@ class DynamoDbStreamsEventSourcePollerTest {private io.github.hectorvent.floci.services.sqs.SqsService sqsService;private io.github.hectorvent.floci.services.sns.SnsService snsService;private io.github.hectorvent.floci.services.s3.S3Service s3Service;+ private final AtomicLong clock = new AtomicLong();@InjectInstance<RequestContext> requestContextInstance;@BeforeEachvoid setUp() {+ clock.set(1_800_000_000_000L);config = mock(EmulatorConfig.class);EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);EmulatorConfig.LambdaServiceConfig lambdaConfig = mock(EmulatorConfig.LambdaServiceConfig.class);@@ -113,7 +116,7 @@ class DynamoDbStreamsEventSourcePollerTest {// the tests drive pollAndInvoke deterministically.poller = new DynamoDbStreamsEventSourcePoller(mock(Vertx.class), streamService, executorService, functionStore,- esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);+ esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);}private EventSourceMapping persistedStreamsEsmWithStaleCheckpoint() {@@ -187,7 +190,11 @@ class DynamoDbStreamsEventSourcePollerTest {private DynamoDbStreamsEventSourcePoller pollerWith(EsmStore store) {return new DynamoDbStreamsEventSourcePoller(mock(Vertx.class), streamService, executorService, functionStore,- store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);+ store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);+ }++ private void advancePastRetry(DynamoDbStreamsEventSourcePoller poller) {+ clock.addAndGet(poller.retryBackoffMs(1));}/**@@ -319,6 +326,7 @@ class DynamoDbStreamsEventSourcePollerTest {p.pollAndInvoke(esm);verify(executorService, timeout(2000)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ advancePastRetry(p);awaitPollCompletedViaSecondFetch(p, esm);verify(store, never()).saveForAccount(anyString(), any());@@ -631,6 +639,7 @@ class DynamoDbStreamsEventSourcePollerTest {verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());// Second poll: attempt 2 (retry 1, which exceeds maxRetryAttempts=1 -> exhausted!)+ advancePastRetry(p);p.pollAndInvoke(esm);// Checkpoint advanced to "s1"@@ -906,6 +915,7 @@ class DynamoDbStreamsEventSourcePollerTest {Thread.sleep(25);}+ advancePastRetry(p);p.pollAndInvoke(esm);verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());@@ -965,6 +975,7 @@ class DynamoDbStreamsEventSourcePollerTest {verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());// Second poll: attempt 2 (retry 1, which exceeds maxRetryAttempts=1 -> exhausted!)+ advancePastRetry(p);p.pollAndInvoke(esm);// Checkpoint advanced to "s1"@@ -1100,6 +1111,7 @@ class DynamoDbStreamsEventSourcePollerTest {}// Second poll: fetches [s2], s2 fails again -> maxRetries=1 exhausts (attempt 2, retry 1)+ advancePastRetry(p);p.pollAndInvoke(esm);verify(store, timeout(2000).times(2)).saveForAccount(eq(ACCOUNT_ID), any());@@ -1120,6 +1132,117 @@ class DynamoDbStreamsEventSourcePollerTest {verify(executorService, times(2)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));}+ @Test+ void failingBatchIsNotReinvokedUntilItsBackoffElapses() throws Exception {+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}")));+ InvokeResult error = new InvokeResult();+ error.setFunctionError("Unhandled");+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(error);++ EventSourceMapping esm = filterEsm();+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));++ p.pollAndInvoke(esm);+ awaitPollCompleted(p);+ p.pollAndInvoke(esm);+ awaitPollCompleted(p);+ verify(executorService, times(1)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));++ advancePastRetry(p);+ p.pollAndInvoke(esm);+ verify(executorService, timeout(2000).times(2)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void retryBackoffDoublesFromThePollIntervalAndCapsOut() {+ assertEquals(2_000, poller.retryBackoffMs(1));+ assertEquals(4_000, poller.retryBackoffMs(2));+ assertEquals(DynamoDbStreamsEventSourcePoller.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));+ }++ @Test+ void batchOlderThan24HoursWithUnlimitedMaximumRecordAgeIsRetried() throws Exception {+ EventSourceMapping esm = filterEsm();+ esm.setMaximumRecordAgeInSeconds(-1);++ assertOldBatchIsRetriedWithoutAnAgeCutoff(esm);+ }++ @Test+ void batchOlderThan24HoursWithUnsetMaximumRecordAgeIsRetried() throws Exception {+ EventSourceMapping esm = filterEsm();++ assertOldBatchIsRetriedWithoutAnAgeCutoff(esm);+ }++ private void assertOldBatchIsRetriedWithoutAnAgeCutoff(EventSourceMapping esm) throws Exception {+ DynamoDbStreamRecord record = ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}");+ record.setApproximateCreationDateTime(clock.get() / 1_000 - 86_401);+ stubTrimHorizon(List.of(record));+ InvokeResult error = new InvokeResult();+ error.setFunctionError("Unhandled");+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(error);++ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();+ destinationConfig.setOnFailure(onFailure);+ esm.setDestinationConfig(destinationConfig);++ EsmStore store = mock(EsmStore.class);+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);++ p.pollAndInvoke(esm);+ awaitPollCompleted(p);++ verify(store, never()).saveForAccount(anyString(), any());+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());++ advancePastRetry(p);+ p.pollAndInvoke(esm);+ verify(executorService, timeout(2000).times(2))+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));+ }++ @Test+ void batchOlderThanMaximumRecordAgeIsDiscardedAndDeliveredToOnFailure() throws Exception {+ DynamoDbStreamRecord record = ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}");+ record.setApproximateCreationDateTime(clock.get() / 1_000 - 61);+ stubTrimHorizon(List.of(record));+ InvokeResult error = new InvokeResult();+ error.setFunctionError("Unhandled");+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(error);++ EventSourceMapping esm = filterEsm();+ esm.setMaximumRecordAgeInSeconds(60);+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();+ destinationConfig.setOnFailure(onFailure);+ esm.setDestinationConfig(destinationConfig);++ EsmStore store = mock(EsmStore.class);+ pollerWith(store).pollAndInvoke(esm);++ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));+ ArgumentCaptor<String> bodyCaptor = ArgumentCaptor.forClass(String.class);+ verify(sqsService, timeout(2000)).sendMessage(anyString(), bodyCaptor.capture(), anyInt(), anyString());+ JsonNode payload = OBJECT_MAPPER.readTree(bodyCaptor.getValue());+ assertEquals("MaximumRecordAgeExceeded", payload.path("requestContext").path("condition").asText());+ }++ private void awaitPollCompleted(DynamoDbStreamsEventSourcePoller poller) throws InterruptedException {+ long deadline = System.currentTimeMillis() + 3_000;+ while (!poller.activePolls.isEmpty() && System.currentTimeMillis() < deadline) {+ Thread.sleep(25);+ }+ assertTrue(poller.activePolls.isEmpty(), "poll did not complete before the test timeout");+ }+private JsonNode readRecords(byte[] payload) {try {return OBJECT_MAPPER.readTree(payload).path("Records");diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.javaindex a12d82e6a2..33452663d0 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java@@ -224,6 +224,7 @@ class EsmIntegrationTest {{"BisectBatchOnFunctionError": true,"MaximumRetryAttempts": 3,+ "MaximumRecordAgeInSeconds": 120,"DestinationConfig": {"OnFailure": {"Destination": "%s"@@ -237,6 +238,7 @@ class EsmIntegrationTest {.statusCode(202).body("BisectBatchOnFunctionError", equalTo(true)).body("MaximumRetryAttempts", equalTo(3))+ .body("MaximumRecordAgeInSeconds", equalTo(120)).body("DestinationConfig.OnFailure.Destination", equalTo(destinationArn));given()@@ -246,6 +248,7 @@ class EsmIntegrationTest {.statusCode(200).body("BisectBatchOnFunctionError", equalTo(true)).body("MaximumRetryAttempts", equalTo(3))+ .body("MaximumRecordAgeInSeconds", equalTo(120)).body("DestinationConfig.OnFailure.Destination", equalTo(destinationArn));given()@@ -254,6 +257,23 @@ class EsmIntegrationTest {.statusCode(202);}+ @Test+ void createEventSourceMappingRejectsMaximumRecordAgeBelowMinimum() {+ given()+ .contentType("application/json")+ .body("""+ {+ "FunctionName": "%s",+ "EventSourceArn": "%s",+ "MaximumRecordAgeInSeconds": 59+ }+ """.formatted(FUNCTION_NAME, QUEUE_ARN))+ .when()+ .post(LAMBDA_BASE + "/event-source-mappings")+ .then()+ .statusCode(400);+ }+@Testvoid responseOmitsFailureConfigWhenUnset() {String uuid = given()@@ -270,6 +290,7 @@ class EsmIntegrationTest {.statusCode(202).body("$", not(hasKey("BisectBatchOnFunctionError"))).body("$", not(hasKey("MaximumRetryAttempts")))+ .body("$", not(hasKey("MaximumRecordAgeInSeconds"))).body("$", not(hasKey("DestinationConfig"))).extract().path("UUID");