Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): back off DynamoDB Streams retries and expire aged records (#3613)

Chris Jovanovski86072aaa8438
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index 92fab8888c..c36705bdf6 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -753,6 +753,8 @@ aws lambda update-function-code \
Connect Lambda to SQS, Kinesis, or DynamoDB Streams. Self-managed Apache Kafka event source mappings are accepted, validated, persisted, and returned on the wire, but Floci does not run an active Kafka consumer poller:
+For DynamoDB Streams mappings, Floci retries failed batches with exponential backoff, honors `MaximumRetryAttempts` and `MaximumRecordAgeInSeconds`, and sends discarded batches to configured SQS or SNS `DestinationConfig.OnFailure` destinations.
+
```bash
# SQS trigger
QUEUE_ARN=$(aws sqs get-queue-attributes \
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index d5d1177da4..0f2afc64a6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -39,6 +39,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.function.LongSupplier;
@ApplicationScoped
public class DynamoDbStreamsEventSourcePoller implements Resettable {
@@ -51,6 +52,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
/** Raised by the stream when a stored checkpoint has aged out of the retained window. */
private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";
+ static final long MAX_RETRY_BACKOFF_MS = 60_000;
private final Vertx vertx;
private final DynamoDbStreamService streamService;
@@ -64,9 +66,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private final S3Service s3Service;
private final String baseUrl;
private final long pollIntervalMs;
+ private final LongSupplier clock;
private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();
final ConcurrentHashMap<String, Boolean> activePolls = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> retryCounts = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, Long> retryNotBefore = new ConcurrentHashMap<>();
private final ExecutorService pollExecutor = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "dynamodb-streams-esm-poller");
t.setDaemon(true);
@@ -84,6 +88,21 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
SqsService sqsService,
SnsService snsService,
S3Service s3Service) {
+ this(vertx, streamService, executorService, functionStore, esmStore, objectMapper, config,
+ filterMatcher, sqsService, snsService, s3Service, System::currentTimeMillis);
+ }
+
+ DynamoDbStreamsEventSourcePoller(Vertx vertx, DynamoDbStreamService streamService,
+ LambdaExecutorService executorService,
+ LambdaFunctionStore functionStore,
+ EsmStore esmStore,
+ ObjectMapper objectMapper,
+ EmulatorConfig config,
+ PipesFilterMatcher filterMatcher,
+ SqsService sqsService,
+ SnsService snsService,
+ S3Service s3Service,
+ LongSupplier clock) {
this.vertx = vertx;
this.streamService = streamService;
this.executorService = executorService;
@@ -96,6 +115,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
this.sqsService = sqsService;
this.snsService = snsService;
this.s3Service = s3Service;
+ this.clock = clock;
}
public void startPersistedPollers() {
@@ -145,6 +165,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
timerIds.clear();
activePolls.clear();
retryCounts.clear();
+ retryNotBefore.clear();
}
public void startPolling(EventSourceMapping esm) {
@@ -170,6 +191,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
LOG.debugv("Stopped DynamoDB Streams polling for ESM {0}", uuid);
}
retryCounts.keySet().removeIf(k -> k.startsWith(uuid + ":"));
+ retryNotBefore.keySet().removeIf(k -> k.startsWith(uuid + ":"));
}
@@ -241,6 +263,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return;
}
+ String checkpointSeq = (lastSeq == null || checkpointTrimmed) ? "TRIM_HORIZON" : lastSeq;
+ String batchKey = esm.getUuid() + ":" + shardId + ":" + checkpointSeq;
+ long now = clock.getAsLong();
+ if (now < retryNotBefore.getOrDefault(batchKey, 0L)) {
+ return;
+ }
+
LOG.infov("DynamoDB Streams ESM {0}: delivering {1} of {2} record(s) to {3}",
esm.getUuid(), matched.size(), records.size(), esm.getFunctionName());
@@ -257,14 +286,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
throw e;
}
- String checkpointSeq = (lastSeq == null || checkpointTrimmed) ? "TRIM_HORIZON" : lastSeq;
- String batchKey = esm.getUuid() + ":" + shardId + ":" + checkpointSeq;
CheckpointOutcome outcome = invokeResult.getFunctionError() == null
? successfulInvocationCheckpoint(esm, invokeResult, lastSeq, records, matched)
: new CheckpointOutcome(null, 0);
if (outcome.lowestFailedIndex() == records.size()) {
retryCounts.remove(batchKey);
+ retryNotBefore.remove(batchKey);
advanceCheckpoint(esm, shardId, outcome.checkpoint());
} else if (outcome.lowestFailedIndex() > 0) {
retryCounts.remove(batchKey);
@@ -273,28 +301,39 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
String nextBatchKey = esm.getUuid() + ":" + shardId + ":" + nextCheckpoint;
Integer maxRetries = esm.getMaximumRetryAttempts();
int currentRetries = retryCounts.merge(nextBatchKey, 1, Integer::sum);
+ Set<String> deliveredSeqs = new HashSet<>();
+ for (DynamoDbStreamRecord rec : matched) {
+ deliveredSeqs.add(rec.getSequenceNumber());
+ }
+ List<DynamoDbStreamRecord> failedRecords = new ArrayList<>();
+ for (int i = outcome.lowestFailedIndex(); i < records.size(); i++) {
+ DynamoDbStreamRecord rec = records.get(i);
+ if (deliveredSeqs.contains(rec.getSequenceNumber())) {
+ failedRecords.add(rec);
+ }
+ }
+ if (failedRecords.isEmpty()) {
+ failedRecords = records.subList(outcome.lowestFailedIndex(), records.size());
+ }
- if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
+ if (hasExceededMaximumRecordAge(esm, failedRecords, now)) {
+ LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
+ esm.getUuid(), newestFetchedSeq);
+ sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,
+ "MaximumRecordAgeExceeded");
+ retryCounts.remove(nextBatchKey);
+ retryNotBefore.remove(nextBatchKey);
+ advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
esm.getUuid(), maxRetries, newestFetchedSeq);
- Set<String> deliveredSeqs = new HashSet<>();
- for (DynamoDbStreamRecord rec : matched) {
- deliveredSeqs.add(rec.getSequenceNumber());
- }
- List<DynamoDbStreamRecord> failedRecords = new ArrayList<>();
- for (int i = outcome.lowestFailedIndex(); i < records.size(); i++) {
- DynamoDbStreamRecord rec = records.get(i);
- if (deliveredSeqs.contains(rec.getSequenceNumber())) {
- failedRecords.add(rec);
- }
- }
- if (failedRecords.isEmpty()) {
- failedRecords = records.subList(outcome.lowestFailedIndex(), records.size());
- }
- sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries);
+ sendToOnFailureDestination(esm, shardId, failedRecords, invokeResult, currentRetries,
+ "RetryAttemptsExhausted");
retryCounts.remove(nextBatchKey);
+ retryNotBefore.remove(nextBatchKey);
advanceCheckpoint(esm, shardId, newestFetchedSeq);
} else {
+ retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));
LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",
esm.getUuid(), currentRetries);
advanceCheckpoint(esm, shardId, nextCheckpoint);
@@ -302,13 +341,24 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
} else {
Integer maxRetries = esm.getMaximumRetryAttempts();
int currentRetries = retryCounts.merge(batchKey, 1, Integer::sum);
- if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
+ if (hasExceededMaximumRecordAge(esm, matched, now)) {
+ LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
+ esm.getUuid(), newestFetchedSeq);
+ sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,
+ "MaximumRecordAgeExceeded");
+ retryCounts.remove(batchKey);
+ retryNotBefore.remove(batchKey);
+ advanceCheckpoint(esm, shardId, newestFetchedSeq);
+ } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
esm.getUuid(), maxRetries, newestFetchedSeq);
- sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries);
+ sendToOnFailureDestination(esm, shardId, matched, invokeResult, currentRetries,
+ "RetryAttemptsExhausted");
retryCounts.remove(batchKey);
+ retryNotBefore.remove(batchKey);
advanceCheckpoint(esm, shardId, newestFetchedSeq);
} else {
+ retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));
String error = invokeResult.getFunctionError() != null
? invokeResult.getFunctionError()
: "batchItemFailures";
@@ -399,10 +449,34 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return previousCheckpoint;
}
+ private boolean hasExceededMaximumRecordAge(EventSourceMapping esm, List<DynamoDbStreamRecord> records,
+ long now) {
+ Integer configuredAge = esm.getMaximumRecordAgeInSeconds();
+ if (configuredAge == null || configuredAge <= 0) {
+ return false;
+ }
+
+ long oldestRecordSeconds = records.stream()
+ .mapToLong(DynamoDbStreamRecord::getApproximateCreationDateTime)
+ .filter(seconds -> seconds > 0)
+ .min()
+ .orElse(0);
+ if (oldestRecordSeconds == 0) {
+ return false;
+ }
+ return now - oldestRecordSeconds * 1_000 >= configuredAge * 1_000L;
+ }
+
+ long retryBackoffMs(int retries) {
+ int doublings = Math.min(retries, 16);
+ return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);
+ }
+
private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,
List<DynamoDbStreamRecord> records,
InvokeResult invokeResult,
- int invokeCount) {
+ int invokeCount,
+ String condition) {
if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {
return;
}
@@ -415,12 +489,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
if (destinationArn.contains(":sqs:")) {
String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);
- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);
+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
sqsService.sendMessage(queueUrl, payload, 0, region);
LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SQS DLQ {1}", esm.getUuid(), destinationArn);
} else if (destinationArn.contains(":sns:")) {
String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);
+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
snsService.publish(destinationArn, null, payload, "ESM OnFailure", region);
LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SNS DLQ {1}", esm.getUuid(), destinationArn);
} else if (destinationArn.contains(":s3:")) {
@@ -429,7 +503,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);
}
String key = buildS3OnFailureKey(esm.getUuid(), shardId, Instant.now(), UUID.randomUUID());
- String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount);
+ String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
RequestScopes.runAs(esm.getAccountId(), () ->
s3Service.putObject(arn.resource(), key,
s3Payload.getBytes(StandardCharsets.UTF_8),
@@ -449,10 +523,11 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,
List<DynamoDbStreamRecord> records,
InvokeResult invokeResult,
- int invokeCount) {
+ int invokeCount,
+ String condition) {
try {
ObjectNode root = (ObjectNode) objectMapper.readTree(
- buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount));
+ buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition));
root.put("payload", buildDynamoDbEvent(records, esm));
return objectMapper.writeValueAsString(root);
} catch (Exception e) {
@@ -469,7 +544,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private String buildOnFailurePayload(EventSourceMapping esm, String shardId,
List<DynamoDbStreamRecord> records,
InvokeResult invokeResult,
- int invokeCount) {
+ int invokeCount,
+ String condition) {
try {
ObjectNode root = objectMapper.createObjectNode();
root.put("version", "1.0");
@@ -478,7 +554,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
ObjectNode requestContext = root.putObject("requestContext");
requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");
requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");
- requestContext.put("condition", "RetryAttemptsExhausted");
+ requestContext.put("condition", condition);
requestContext.put("approximateInvokeCount", invokeCount);
ObjectNode responseContext = root.putObject("responseContext");
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
index b2cf08c25f..50b12dfd19 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaController.java
@@ -372,6 +372,9 @@ public class LambdaController {
if (esm.getMaximumRetryAttempts() != null) {
node.put("MaximumRetryAttempts", esm.getMaximumRetryAttempts());
}
+ if (esm.getMaximumRecordAgeInSeconds() != null) {
+ node.put("MaximumRecordAgeInSeconds", esm.getMaximumRecordAgeInSeconds());
+ }
if (esm.getDestinationConfig() != null && esm.getDestinationConfig().getOnFailure() != null) {
ObjectNode destinationConfig = node.putObject("DestinationConfig");
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
index 5ca0725cdc..a92d8a5885 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
@@ -1320,6 +1320,7 @@ public class LambdaService implements ResourceProvider {
: null;
Integer maximumRetryAttempts = parseMaximumRetryAttempts(request);
+ Integer maximumRecordAgeInSeconds = parseMaximumRecordAgeInSeconds(request);
EventSourceMapping.DestinationConfig destinationConfig = parseDestinationConfig(request);
@@ -1347,6 +1348,7 @@ public class LambdaService implements ResourceProvider {
esm.setFunctionResponseTypes(functionResponseTypes);
esm.setBisectBatchOnFunctionError(bisectBatchOnFunctionError);
esm.setMaximumRetryAttempts(maximumRetryAttempts);
+ esm.setMaximumRecordAgeInSeconds(maximumRecordAgeInSeconds);
esm.setDestinationConfig(destinationConfig);
esm.setFilterCriteria(filterCriteria);
esm.setStartingPosition(startingPosition.position());
@@ -1752,6 +1754,28 @@ public class LambdaService implements ResourceProvider {
return (int) value;
}
+ private Integer parseMaximumRecordAgeInSeconds(Map<String, Object> request) {
+ Object raw = request.get("MaximumRecordAgeInSeconds");
+ if (raw == null) {
+ return null;
+ }
+ if (!(raw instanceof Number)) {
+ throw new AwsException("InvalidParameterValueException",
+ "MaximumRecordAgeInSeconds must be a numeric value", 400);
+ }
+ double d = ((Number) raw).doubleValue();
+ if (Double.isNaN(d) || Double.isInfinite(d) || d != Math.floor(d)) {
+ throw new AwsException("InvalidParameterValueException",
+ "MaximumRecordAgeInSeconds must be an integer", 400);
+ }
+ long value = ((Number) raw).longValue();
+ if (value != -1 && (value < 60 || value > 604800)) {
+ throw new AwsException("InvalidParameterValueException",
+ "MaximumRecordAgeInSeconds must be -1 or between 60 and 604800 (got " + value + ")", 400);
+ }
+ return (int) value;
+ }
+
private void startPollingHelper(EventSourceMapping esm) {
if (esm.getEventSourceArn() == null) {
return;
@@ -1826,6 +1850,9 @@ public class LambdaService implements ResourceProvider {
if (request.containsKey("MaximumRetryAttempts")) {
esm.setMaximumRetryAttempts(parseMaximumRetryAttempts(request));
}
+ if (request.containsKey("MaximumRecordAgeInSeconds")) {
+ esm.setMaximumRecordAgeInSeconds(parseMaximumRecordAgeInSeconds(request));
+ }
if (request.containsKey("DestinationConfig")) {
esm.setDestinationConfig(parseDestinationConfig(request));
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
index 32e999cf92..b3662043b6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/EventSourceMapping.java
@@ -32,6 +32,7 @@ public class EventSourceMapping {
private ScalingConfig scalingConfig;
private Boolean bisectBatchOnFunctionError;
private Integer maximumRetryAttempts;
+ private Integer maximumRecordAgeInSeconds;
private DestinationConfig destinationConfig;
private FilterCriteria filterCriteria;
private Map<String, Object> selfManagedEventSource;
@@ -127,6 +128,14 @@ public class EventSourceMapping {
this.maximumRetryAttempts = maximumRetryAttempts;
}
+ public Integer getMaximumRecordAgeInSeconds() {
+ return maximumRecordAgeInSeconds;
+ }
+
+ public void setMaximumRecordAgeInSeconds(Integer maximumRecordAgeInSeconds) {
+ this.maximumRecordAgeInSeconds = maximumRecordAgeInSeconds;
+ }
+
public DestinationConfig getDestinationConfig() {
return destinationConfig;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index dad0016048..52d9d61fbd 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -42,6 +42,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -86,12 +87,14 @@ class DynamoDbStreamsEventSourcePollerTest {
private io.github.hectorvent.floci.services.sqs.SqsService sqsService;
private io.github.hectorvent.floci.services.sns.SnsService snsService;
private io.github.hectorvent.floci.services.s3.S3Service s3Service;
+ private final AtomicLong clock = new AtomicLong();
@Inject
Instance<RequestContext> requestContextInstance;
@BeforeEach
void setUp() {
+ clock.set(1_800_000_000_000L);
config = mock(EmulatorConfig.class);
EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);
EmulatorConfig.LambdaServiceConfig lambdaConfig = mock(EmulatorConfig.LambdaServiceConfig.class);
@@ -113,7 +116,7 @@ class DynamoDbStreamsEventSourcePollerTest {
// the tests drive pollAndInvoke deterministically.
poller = new DynamoDbStreamsEventSourcePoller(
mock(Vertx.class), streamService, executorService, functionStore,
- esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);
+ esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
}
private EventSourceMapping persistedStreamsEsmWithStaleCheckpoint() {
@@ -187,7 +190,11 @@ class DynamoDbStreamsEventSourcePollerTest {
private DynamoDbStreamsEventSourcePoller pollerWith(EsmStore store) {
return new DynamoDbStreamsEventSourcePoller(
mock(Vertx.class), streamService, executorService, functionStore,
- store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);
+ store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
+ }
+
+ private void advancePastRetry(DynamoDbStreamsEventSourcePoller poller) {
+ clock.addAndGet(poller.retryBackoffMs(1));
}
/**
@@ -319,6 +326,7 @@ class DynamoDbStreamsEventSourcePollerTest {
p.pollAndInvoke(esm);
verify(executorService, timeout(2000)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ advancePastRetry(p);
awaitPollCompletedViaSecondFetch(p, esm);
verify(store, never()).saveForAccount(anyString(), any());
@@ -631,6 +639,7 @@ class DynamoDbStreamsEventSourcePollerTest {
verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
// Second poll: attempt 2 (retry 1, which exceeds maxRetryAttempts=1 -> exhausted!)
+ advancePastRetry(p);
p.pollAndInvoke(esm);
// Checkpoint advanced to "s1"
@@ -906,6 +915,7 @@ class DynamoDbStreamsEventSourcePollerTest {
Thread.sleep(25);
}
+ advancePastRetry(p);
p.pollAndInvoke(esm);
verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
@@ -965,6 +975,7 @@ class DynamoDbStreamsEventSourcePollerTest {
verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
// Second poll: attempt 2 (retry 1, which exceeds maxRetryAttempts=1 -> exhausted!)
+ advancePastRetry(p);
p.pollAndInvoke(esm);
// Checkpoint advanced to "s1"
@@ -1100,6 +1111,7 @@ class DynamoDbStreamsEventSourcePollerTest {
}
// Second poll: fetches [s2], s2 fails again -> maxRetries=1 exhausts (attempt 2, retry 1)
+ advancePastRetry(p);
p.pollAndInvoke(esm);
verify(store, timeout(2000).times(2)).saveForAccount(eq(ACCOUNT_ID), any());
@@ -1120,6 +1132,117 @@ class DynamoDbStreamsEventSourcePollerTest {
verify(executorService, times(2)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
}
+ @Test
+ void failingBatchIsNotReinvokedUntilItsBackoffElapses() throws Exception {
+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}")));
+ InvokeResult error = new InvokeResult();
+ error.setFunctionError("Unhandled");
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(error);
+
+ EventSourceMapping esm = filterEsm();
+ DynamoDbStreamsEventSourcePoller p = pollerWith(mock(EsmStore.class));
+
+ p.pollAndInvoke(esm);
+ awaitPollCompleted(p);
+ p.pollAndInvoke(esm);
+ awaitPollCompleted(p);
+ verify(executorService, times(1)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+
+ advancePastRetry(p);
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000).times(2)).invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void retryBackoffDoublesFromThePollIntervalAndCapsOut() {
+ assertEquals(2_000, poller.retryBackoffMs(1));
+ assertEquals(4_000, poller.retryBackoffMs(2));
+ assertEquals(DynamoDbStreamsEventSourcePoller.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));
+ }
+
+ @Test
+ void batchOlderThan24HoursWithUnlimitedMaximumRecordAgeIsRetried() throws Exception {
+ EventSourceMapping esm = filterEsm();
+ esm.setMaximumRecordAgeInSeconds(-1);
+
+ assertOldBatchIsRetriedWithoutAnAgeCutoff(esm);
+ }
+
+ @Test
+ void batchOlderThan24HoursWithUnsetMaximumRecordAgeIsRetried() throws Exception {
+ EventSourceMapping esm = filterEsm();
+
+ assertOldBatchIsRetriedWithoutAnAgeCutoff(esm);
+ }
+
+ private void assertOldBatchIsRetriedWithoutAnAgeCutoff(EventSourceMapping esm) throws Exception {
+ DynamoDbStreamRecord record = ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}");
+ record.setApproximateCreationDateTime(clock.get() / 1_000 - 86_401);
+ stubTrimHorizon(List.of(record));
+ InvokeResult error = new InvokeResult();
+ error.setFunctionError("Unhandled");
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(error);
+
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+
+ EsmStore store = mock(EsmStore.class);
+ DynamoDbStreamsEventSourcePoller p = pollerWith(store);
+
+ p.pollAndInvoke(esm);
+ awaitPollCompleted(p);
+
+ verify(store, never()).saveForAccount(anyString(), any());
+ verify(sqsService, never()).sendMessage(anyString(), anyString(), anyInt(), anyString());
+
+ advancePastRetry(p);
+ p.pollAndInvoke(esm);
+ verify(executorService, timeout(2000).times(2))
+ .invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse));
+ }
+
+ @Test
+ void batchOlderThanMaximumRecordAgeIsDiscardedAndDeliveredToOnFailure() throws Exception {
+ DynamoDbStreamRecord record = ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}");
+ record.setApproximateCreationDateTime(clock.get() / 1_000 - 61);
+ stubTrimHorizon(List.of(record));
+ InvokeResult error = new InvokeResult();
+ error.setFunctionError("Unhandled");
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(error);
+
+ EventSourceMapping esm = filterEsm();
+ esm.setMaximumRecordAgeInSeconds(60);
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination("arn:aws:sqs:us-east-1:000000000000:my-dlq");
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+
+ EsmStore store = mock(EsmStore.class);
+ pollerWith(store).pollAndInvoke(esm);
+
+ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());
+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));
+ ArgumentCaptor<String> bodyCaptor = ArgumentCaptor.forClass(String.class);
+ verify(sqsService, timeout(2000)).sendMessage(anyString(), bodyCaptor.capture(), anyInt(), anyString());
+ JsonNode payload = OBJECT_MAPPER.readTree(bodyCaptor.getValue());
+ assertEquals("MaximumRecordAgeExceeded", payload.path("requestContext").path("condition").asText());
+ }
+
+ private void awaitPollCompleted(DynamoDbStreamsEventSourcePoller poller) throws InterruptedException {
+ long deadline = System.currentTimeMillis() + 3_000;
+ while (!poller.activePolls.isEmpty() && System.currentTimeMillis() < deadline) {
+ Thread.sleep(25);
+ }
+ assertTrue(poller.activePolls.isEmpty(), "poll did not complete before the test timeout");
+ }
+
private JsonNode readRecords(byte[] payload) {
try {
return OBJECT_MAPPER.readTree(payload).path("Records");
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
index a12d82e6a2..33452663d0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/EsmIntegrationTest.java
@@ -224,6 +224,7 @@ class EsmIntegrationTest {
{
"BisectBatchOnFunctionError": true,
"MaximumRetryAttempts": 3,
+ "MaximumRecordAgeInSeconds": 120,
"DestinationConfig": {
"OnFailure": {
"Destination": "%s"
@@ -237,6 +238,7 @@ class EsmIntegrationTest {
.statusCode(202)
.body("BisectBatchOnFunctionError", equalTo(true))
.body("MaximumRetryAttempts", equalTo(3))
+ .body("MaximumRecordAgeInSeconds", equalTo(120))
.body("DestinationConfig.OnFailure.Destination", equalTo(destinationArn));
given()
@@ -246,6 +248,7 @@ class EsmIntegrationTest {
.statusCode(200)
.body("BisectBatchOnFunctionError", equalTo(true))
.body("MaximumRetryAttempts", equalTo(3))
+ .body("MaximumRecordAgeInSeconds", equalTo(120))
.body("DestinationConfig.OnFailure.Destination", equalTo(destinationArn));
given()
@@ -254,6 +257,23 @@ class EsmIntegrationTest {
.statusCode(202);
}
+ @Test
+ void createEventSourceMappingRejectsMaximumRecordAgeBelowMinimum() {
+ given()
+ .contentType("application/json")
+ .body("""
+ {
+ "FunctionName": "%s",
+ "EventSourceArn": "%s",
+ "MaximumRecordAgeInSeconds": 59
+ }
+ """.formatted(FUNCTION_NAME, QUEUE_ARN))
+ .when()
+ .post(LAMBDA_BASE + "/event-source-mappings")
+ .then()
+ .statusCode(400);
+ }
+
@Test
void responseOmitsFailureConfigWhenUnset() {
String uuid = given()
@@ -270,6 +290,7 @@ class EsmIntegrationTest {
.statusCode(202)
.body("$", not(hasKey("BisectBatchOnFunctionError")))
.body("$", not(hasKey("MaximumRetryAttempts")))
+ .body("$", not(hasKey("MaximumRecordAgeInSeconds")))
.body("$", not(hasKey("DestinationConfig")))
.extract()
.path("UUID");