Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): share the stream OnFailure delivery and send it in the destination's account (#6224)

Attila Kisb0a4d5064bd0
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 8f41eed54a..0dabf3889f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -5,9 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
-import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
-import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
@@ -16,26 +14,21 @@ import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.pipes.PipesFilterMatcher;
-import io.github.hectorvent.floci.services.s3.S3Service;
-import io.github.hectorvent.floci.services.sns.SnsService;
-import io.github.hectorvent.floci.services.sqs.SqsService;
import io.vertx.core.Vertx;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
-import java.nio.charset.StandardCharsets;
import java.time.Instant;
-import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.OptionalLong;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -46,14 +39,10 @@ import java.util.stream.Collectors;
public class DynamoDbStreamsEventSourcePoller implements Resettable {
private static final Logger LOG = Logger.getLogger(DynamoDbStreamsEventSourcePoller.class);
- private static final DateTimeFormatter S3_ON_FAILURE_PATH_DATE_FORMATTER =
- DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);
- private static final DateTimeFormatter S3_ON_FAILURE_FILE_TIME_FORMATTER =
- DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH.mm.ss").withZone(ZoneOffset.UTC);
+ private static final String BATCH_INFO_NAME = "DDBStreamBatchInfo";
/** Raised by the stream when a stored checkpoint has aged out of the retained window. */
private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";
- static final long MAX_RETRY_BACKOFF_MS = 60_000;
private final Vertx vertx;
private final DynamoDbStreamReader streamReader;
@@ -62,10 +51,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private final EsmStore esmStore;
private final ObjectMapper objectMapper;
private final PipesFilterMatcher filterMatcher;
- private final SqsService sqsService;
- private final SnsService snsService;
- private final S3Service s3Service;
- private final String baseUrl;
+ private final StreamBatchFailures batchFailures;
private final long pollIntervalMs;
private final LongSupplier clock;
private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();
@@ -92,11 +78,9 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
ObjectMapper objectMapper,
EmulatorConfig config,
PipesFilterMatcher filterMatcher,
- SqsService sqsService,
- SnsService snsService,
- S3Service s3Service) {
+ StreamBatchFailures batchFailures) {
this(vertx, streamReader, executorService, targetResolver, esmStore, objectMapper, config,
- filterMatcher, sqsService, snsService, s3Service, System::currentTimeMillis);
+ filterMatcher, batchFailures, System::currentTimeMillis);
}
DynamoDbStreamsEventSourcePoller(Vertx vertx, DynamoDbStreamReader streamReader,
@@ -106,9 +90,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
ObjectMapper objectMapper,
EmulatorConfig config,
PipesFilterMatcher filterMatcher,
- SqsService sqsService,
- SnsService snsService,
- S3Service s3Service,
+ StreamBatchFailures batchFailures,
LongSupplier clock) {
this.vertx = vertx;
this.streamReader = streamReader;
@@ -117,11 +99,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
this.esmStore = esmStore;
this.objectMapper = objectMapper;
this.pollIntervalMs = config.services().lambda().pollIntervalMs();
- this.baseUrl = config.effectiveBaseUrl();
this.filterMatcher = filterMatcher;
- this.sqsService = sqsService;
- this.snsService = snsService;
- this.s3Service = s3Service;
+ this.batchFailures = batchFailures;
this.clock = clock;
}
@@ -294,7 +273,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private void pollShard(EventSourceMapping esm, LambdaFunction fn, DynamoDbStreamReader.Stream stream,
String shardId) {
String lastSeq = esm.getShardSequenceNumbers().get(shardId);
- String persistedKey = batchStateKey(esm.getUuid(), shardId, lastSeq);
+ String persistedKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, lastSeq);
boolean checkpointTrimmed = false;
DynamoDbStreamReader.RecordsPage page;
@@ -317,7 +296,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
+ "trim horizon; records between were dropped from the stream",
esm.getUuid(), lastSeq);
page = streamReader.readAfter(stream, shardId, null, Math.min(esm.getBatchSize(),
- bisectLimit(esm, batchStateKey(esm.getUuid(), shardId, null))));
+ bisectLimit(esm, StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, null))));
}
List<DynamoDbStreamReader.Record> records = page.records();
@@ -353,7 +332,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
records, filterNodes, filterMatcher.applyFilterCriteria(filterNodes, filterParams));
}
- String batchKey = batchStateKey(esm.getUuid(), shardId, checkpointTrimmed ? null : lastSeq);
+ String batchKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, checkpointTrimmed ? null : lastSeq);
if (matched.isEmpty()) {
clearBatchState(batchKey);
advanceCheckpoint(esm, shardId, newestFetchedSeq);
@@ -397,8 +376,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
clearBatchState(batchKey);
String nextCheckpoint = outcome.checkpoint();
- String nextBatchKey = batchStateKey(esm.getUuid(), shardId, nextCheckpoint);
- Integer maxRetries = esm.getMaximumRetryAttempts();
+ String nextBatchKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, nextCheckpoint);
int currentRetries = retryCounts.merge(nextBatchKey, 1, Integer::sum);
Set<String> deliveredSeqs = new HashSet<>();
for (DynamoDbStreamReader.Record rec : matched) {
@@ -419,12 +397,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
esm.getUuid(), newestFetchedSeq);
dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));
- } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
+ currentRetries, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED, newestFetchedSeq));
+ } else if (StreamBatchFailures.retryAttemptsExhausted(esm, currentRetries)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
- esm.getUuid(), maxRetries, newestFetchedSeq);
+ esm.getUuid(), esm.getMaximumRetryAttempts(), newestFetchedSeq);
dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,
- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));
+ currentRetries, StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED, newestFetchedSeq));
} else {
retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));
LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",
@@ -432,14 +410,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
advanceCheckpoint(esm, shardId, nextCheckpoint);
}
} else {
- Integer maxRetries = esm.getMaximumRetryAttempts();
int currentRetries = retryCounts.getOrDefault(batchKey, 0) + 1;
if (hasExceededMaximumRecordAge(esm, matched, now)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",
esm.getUuid(), newestFetchedSeq);
clearBatchState(batchKey);
dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));
+ currentRetries, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED, newestFetchedSeq));
} else if (invokeResult.getFunctionError() != null
&& Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError()) && matched.size() > 1) {
// ponytail: the halved limit is keyed by checkpoint, so once a good half advances the
@@ -450,12 +427,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
bisectLimits.put(batchKey, limit);
LOG.infov("DynamoDB Streams ESM {0}: function error on {1} record(s), bisecting to {2}",
esm.getUuid(), records.size(), limit);
- } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {
+ } else if (StreamBatchFailures.retryAttemptsExhausted(esm, currentRetries)) {
LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",
- esm.getUuid(), maxRetries, newestFetchedSeq);
+ esm.getUuid(), esm.getMaximumRetryAttempts(), newestFetchedSeq);
clearBatchState(batchKey);
dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,
- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));
+ currentRetries, StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED, newestFetchedSeq));
} else {
retryCounts.put(batchKey, currentRetries);
retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));
@@ -479,8 +456,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
clearBatchState(key);
return;
}
- sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),
- failure.invokeCount(), failure.condition());
+ batchFailures.sendToOnFailureDestination(esm, discardedBatch(esm, shardId, failure.records()),
+ failure.invokeResult(), failure.invokeCount(), failure.condition());
clearBatchState(key);
advanceCheckpoint(esm, shardId, failure.advanceTo());
}
@@ -501,10 +478,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
return uuid + ":" + shardId;
}
- private static String batchStateKey(String uuid, String shardId, String sequence) {
- return uuid + ":" + shardId + ":" + (sequence == null ? "TRIM_HORIZON" : sequence);
- }
-
/**
* Returns the last record that can be consumed after a successful invocation. Floci stores the
* last consumed sequence and resumes with {@code AFTER_SEQUENCE_NUMBER}, so a partial failure
@@ -580,20 +553,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
private boolean hasExceededMaximumRecordAge(EventSourceMapping esm, List<DynamoDbStreamReader.Record> records,
long now) {
- Integer configuredAge = esm.getMaximumRecordAgeInSeconds();
- if (configuredAge == null || configuredAge <= 0) {
- return false;
- }
-
- long oldestRecordSeconds = records.stream()
+ OptionalLong oldestRecordSeconds = records.stream()
.mapToLong(DynamoDbStreamsEventSourcePoller::approximateCreationSeconds)
.filter(seconds -> seconds > 0)
- .min()
- .orElse(0);
- if (oldestRecordSeconds == 0) {
- return false;
- }
- return now - oldestRecordSeconds * 1_000 >= configuredAge * 1_000L;
+ .min();
+ return oldestRecordSeconds.isPresent() && StreamBatchFailures.maximumRecordAgeExceeded(
+ esm, Instant.ofEpochSecond(oldestRecordSeconds.getAsLong()), now);
}
private static long approximateCreationSeconds(DynamoDbStreamReader.Record rec) {
@@ -601,125 +566,22 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
}
long retryBackoffMs(int retries) {
- int doublings = Math.min(retries, 16);
- return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);
+ return StreamBatchFailures.retryBackoffMs(pollIntervalMs, retries);
}
- private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,
- List<DynamoDbStreamReader.Record> records,
- InvokeResult invokeResult,
- int invokeCount,
- String condition) {
- if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {
- return;
- }
- String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();
- if (destinationArn == null || destinationArn.isBlank()) {
- return;
- }
-
- try {
- if (destinationArn.contains(":sqs:")) {
- String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
- String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);
- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
- sqsService.sendMessage(queueUrl, payload, 0, region);
- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SQS DLQ {1}", esm.getUuid(), destinationArn);
- } else if (destinationArn.contains(":sns:")) {
- String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
- snsService.publish(destinationArn, null, payload, "ESM OnFailure", region);
- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SNS DLQ {1}", esm.getUuid(), destinationArn);
- } else if (destinationArn.contains(":s3:")) {
- AwsArnUtils.Arn arn = AwsArnUtils.parse(destinationArn);
- if (!"s3".equals(arn.service()) || arn.resource().isBlank()) {
- throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);
- }
- String key = buildS3OnFailureKey(esm.getUuid(), shardId, Instant.now(), UUID.randomUUID());
- String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);
- RequestScopes.runAs(esm.getAccountId(), () ->
- s3Service.putObject(arn.resource(), key,
- s3Payload.getBytes(StandardCharsets.UTF_8),
- "application/json", Map.of()));
- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to S3 bucket {1} with key {2}",
- esm.getUuid(), arn.resource(), key);
- } else {
- LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}; discarding records",
- esm.getUuid(), destinationArn);
- }
- } catch (Exception e) {
- LOG.errorv("DynamoDB Streams ESM {0}: failed to send to OnFailure destination {1}; discarding records: {2}",
- esm.getUuid(), destinationArn, e.getMessage());
- }
+ private StreamBatchFailures.DiscardedBatch discardedBatch(EventSourceMapping esm, String shardId,
+ List<DynamoDbStreamReader.Record> records) {
+ DynamoDbStreamReader.Record first = records.get(0);
+ DynamoDbStreamReader.Record last = records.get(records.size() - 1);
+ return new StreamBatchFailures.DiscardedBatch(BATCH_INFO_NAME, shardId,
+ first.sequenceNumber(), last.sequenceNumber(),
+ creationTime(first), creationTime(last),
+ records.size(), esm.getEventSourceArn(), () -> buildDynamoDbEvent(records, esm));
}
- private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,
- List<DynamoDbStreamReader.Record> records,
- InvokeResult invokeResult,
- int invokeCount,
- String condition) {
- try {
- ObjectNode root = (ObjectNode) objectMapper.readTree(
- buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition));
- root.put("payload", buildDynamoDbEvent(records, esm));
- return objectMapper.writeValueAsString(root);
- } catch (Exception e) {
- throw new IllegalStateException("Failed to serialize S3 OnFailure payload", e);
- }
- }
-
- static String buildS3OnFailureKey(String esmUuid, String shardId, Instant now, UUID randomId) {
- return "aws/lambda/" + esmUuid + "/" + shardId + "/"
- + S3_ON_FAILURE_PATH_DATE_FORMATTER.format(now) + "/"
- + S3_ON_FAILURE_FILE_TIME_FORMATTER.format(now) + "-" + randomId;
- }
-
- private String buildOnFailurePayload(EventSourceMapping esm, String shardId,
- List<DynamoDbStreamReader.Record> records,
- InvokeResult invokeResult,
- int invokeCount,
- String condition) {
- try {
- ObjectNode root = objectMapper.createObjectNode();
- root.put("version", "1.0");
- root.put("timestamp", DateTimeFormatter.ISO_INSTANT.format(Instant.now()));
-
- ObjectNode requestContext = root.putObject("requestContext");
- requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");
- requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");
- requestContext.put("condition", condition);
- requestContext.put("approximateInvokeCount", invokeCount);
-
- ObjectNode responseContext = root.putObject("responseContext");
- responseContext.put("statusCode", invokeResult.getStatusCode() != 0 ? invokeResult.getStatusCode() : 200);
- responseContext.put("executedVersion", invokeResult.getExecutedVersion() != null ? invokeResult.getExecutedVersion() : "$LATEST");
- if (invokeResult.getFunctionError() != null) {
- responseContext.put("functionError", invokeResult.getFunctionError());
- }
-
- ObjectNode batchInfo = root.putObject("DDBStreamBatchInfo");
- batchInfo.put("shardId", shardId);
- String startSeq = records.isEmpty() ? "" : records.get(0).sequenceNumber();
- String endSeq = records.isEmpty() ? "" : records.get(records.size() - 1).sequenceNumber();
- batchInfo.put("startSequenceNumber", startSeq);
- batchInfo.put("endSequenceNumber", endSeq);
-
- if (!records.isEmpty()) {
- long firstArrival = approximateCreationSeconds(records.get(0));
- long lastArrival = approximateCreationSeconds(records.get(records.size() - 1));
- batchInfo.put("approximateArrivalOfFirstRecord",
- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(firstArrival)));
- batchInfo.put("approximateArrivalOfLastRecord",
- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(lastArrival)));
- }
- batchInfo.put("batchSize", records.size());
- batchInfo.put("streamArn", esm.getEventSourceArn());
-
- return objectMapper.writeValueAsString(root);
- } catch (Exception e) {
- LOG.warnv("Failed to serialize OnFailure payload: {0}", e.getMessage());
- return "{}";
- }
+ /** A stream record's creation time is whole seconds, and AWS writes it without a fraction. */
+ private static String creationTime(DynamoDbStreamReader.Record rec) {
+ return DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(approximateCreationSeconds(rec)));
}
private String buildDynamoDbEvent(List<DynamoDbStreamReader.Record> records, EventSourceMapping esm) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.java b/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.java
new file mode 100644
index 0000000000..2f08e4b7f9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.java
@@ -0,0 +1,188 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+
+/**
+ * What Lambda does with a stream batch that keeps failing: the retry backoff, the
+ * {@code MaximumRetryAttempts} and {@code MaximumRecordAgeInSeconds} limits, and the invocation
+ * record it sends to the mapping's {@code DestinationConfig.OnFailure} when it discards the batch.
+ * Only the batch info block and the event depend on the stream source; its poller passes them in a
+ * {@link DiscardedBatch}.
+ */
+@ApplicationScoped
+public class StreamBatchFailures {
+
+ private static final Logger LOG = Logger.getLogger(StreamBatchFailures.class);
+
+ static final String RETRY_ATTEMPTS_EXHAUSTED = "RetryAttemptsExhausted";
+ static final String MAXIMUM_RECORD_AGE_EXCEEDED = "MaximumRecordAgeExceeded";
+ static final long MAX_RETRY_BACKOFF_MS = 60_000;
+
+ private static final DateTimeFormatter S3_KEY_DATE =
+ DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);
+ private static final DateTimeFormatter S3_KEY_TIME =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH.mm.ss").withZone(ZoneOffset.UTC);
+
+ /**
+ * The source-specific part of the invocation record. {@code batchInfoName} names the source's
+ * batch info block, such as {@code DDBStreamBatchInfo}. The arrival times are written as given,
+ * because their precision depends on the source. {@code event} builds the event of the discarded
+ * records, in the shape the function gets it; only an S3 destination asks for it, as the
+ * {@code payload} string.
+ */
+ record DiscardedBatch(String batchInfoName, String shardId, String startSequenceNumber,
+ String endSequenceNumber, String firstArrival, String lastArrival,
+ int batchSize, String streamArn, Supplier<String> event) {
+ }
+
+ private final SqsService sqsService;
+ private final SnsService snsService;
+ private final S3Service s3Service;
+ private final ObjectMapper objectMapper;
+ private final String baseUrl;
+
+ @Inject
+ public StreamBatchFailures(SqsService sqsService, SnsService snsService, S3Service s3Service,
+ ObjectMapper objectMapper, EmulatorConfig config) {
+ this.sqsService = sqsService;
+ this.snsService = snsService;
+ this.s3Service = s3Service;
+ this.objectMapper = objectMapper;
+ this.baseUrl = config.effectiveBaseUrl();
+ }
+
+ static long retryBackoffMs(long pollIntervalMs, int retries) {
+ int doublings = Math.min(retries, 16);
+ return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);
+ }
+
+ /** {@code invokeCount} is 1 + the retries so far. An unset or {@code -1} limit never runs out. */
+ static boolean retryAttemptsExhausted(EventSourceMapping esm, int invokeCount) {
+ Integer maximum = esm.getMaximumRetryAttempts();
+ return maximum != null && maximum >= 0 && invokeCount > maximum;
+ }
+
+ /** An unset or {@code -1} limit never expires a batch. */
+ static boolean maximumRecordAgeExceeded(EventSourceMapping esm, Instant oldestArrival, long nowMillis) {
+ Integer maximum = esm.getMaximumRecordAgeInSeconds();
+ return maximum != null && maximum > 0 && nowMillis - oldestArrival.toEpochMilli() >= maximum * 1_000L;
+ }
+
+ /** The retry state of a failing batch is keyed by the checkpoint it is read after. */
+ static String batchStateKey(String esmUuid, String shardId, String checkpoint) {
+ return esmUuid + ":" + shardId + ":" + (checkpoint == null ? "TRIM_HORIZON" : checkpoint);
+ }
+
+ static String s3Key(String esmUuid, String shardId, Instant now, UUID randomId) {
+ return "aws/lambda/" + esmUuid + "/" + shardId + "/"
+ + S3_KEY_DATE.format(now) + "/"
+ + S3_KEY_TIME.format(now) + "-" + randomId;
+ }
+
+ /**
+ * Sends the invocation record of a discarded batch to the mapping's OnFailure destination, if it
+ * has one. A refused send is logged and the batch is dropped all the same, as AWS does.
+ */
+ void sendToOnFailureDestination(EventSourceMapping esm, DiscardedBatch batch, InvokeResult invokeResult,
+ int invokeCount, String condition) {
+ if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {
+ return;
+ }
+ String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();
+ if (destinationArn == null || destinationArn.isBlank()) {
+ return;
+ }
+
+ // A poller thread carries no request context, so the destination's account is set before
+ // its account-aware store is read.
+ String destinationAccount = AwsArnUtils.accountOrDefault(destinationArn, esm.getAccountId());
+ try {
+ if (destinationArn.contains(":sqs:")) {
+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
+ String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);
+ String record = invocationRecord(esm, batch, invokeResult, invokeCount, condition, false);
+ RequestScopes.runAs(destinationAccount, () -> sqsService.sendMessage(queueUrl, record, 0, region));
+ } else if (destinationArn.contains(":sns:")) {
+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());
+ String record = invocationRecord(esm, batch, invokeResult, invokeCount, condition, false);
+ RequestScopes.runAs(destinationAccount, () ->
+ snsService.publish(destinationArn, null, record, "ESM OnFailure", region));
+ } else if (destinationArn.contains(":s3:")) {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(destinationArn);
+ if (!"s3".equals(arn.service()) || arn.resource().isBlank()) {
+ throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);
+ }
+ String key = s3Key(esm.getUuid(), batch.shardId(), Instant.now(), UUID.randomUUID());
+ byte[] body = invocationRecord(esm, batch, invokeResult, invokeCount, condition, true)
+ .getBytes(StandardCharsets.UTF_8);
+ RequestScopes.runAs(esm.getAccountId(), () ->
+ s3Service.putObject(arn.resource(), key, body, "application/json", Map.of()));
+ } else {
+ LOG.warnv("ESM {0}: unsupported OnFailure destination {1}, discarding the batch",
+ esm.getUuid(), destinationArn);
+ return;
+ }
+ LOG.infov("ESM {0}: sent the discarded batch of shard {1} to {2}",
+ esm.getUuid(), batch.shardId(), destinationArn);
+ } catch (Exception e) {
+ LOG.errorv(e, "ESM {0}: failed to send to OnFailure destination {1}, discarding the batch",
+ esm.getUuid(), destinationArn);
+ }
+ }
+
+ private String invocationRecord(EventSourceMapping esm, DiscardedBatch batch, InvokeResult invokeResult,
+ int invokeCount, String condition, boolean withPayload)
+ throws JsonProcessingException {
+ ObjectNode root = objectMapper.createObjectNode();
+ ObjectNode requestContext = root.putObject("requestContext");
+ requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");
+ requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");
+ requestContext.put("condition", condition);
+ requestContext.put("approximateInvokeCount", invokeCount);
+
+ ObjectNode responseContext = root.putObject("responseContext");
+ responseContext.put("statusCode", invokeResult.getStatusCode() != 0 ? invokeResult.getStatusCode() : 200);
+ responseContext.put("executedVersion",
+ invokeResult.getExecutedVersion() != null ? invokeResult.getExecutedVersion() : "$LATEST");
+ if (invokeResult.getFunctionError() != null) {
+ responseContext.put("functionError", invokeResult.getFunctionError());
+ }
+
+ root.put("version", "1.0");
+ if (withPayload) {
+ root.put("payload", batch.event().get());
+ }
+ root.put("timestamp", AsyncInvokeDestinationRouter.formatRecordTimestamp(Instant.now()));
+
+ ObjectNode batchInfo = root.putObject(batch.batchInfoName());
+ batchInfo.put("shardId", batch.shardId());
+ batchInfo.put("startSequenceNumber", batch.startSequenceNumber());
+ batchInfo.put("endSequenceNumber", batch.endSequenceNumber());
+ batchInfo.put("approximateArrivalOfFirstRecord", batch.firstArrival());
+ batchInfo.put("approximateArrivalOfLastRecord", batch.lastArrival());
+ batchInfo.put("batchSize", batch.batchSize());
+ batchInfo.put("streamArn", batch.streamArn());
+ return objectMapper.writeValueAsString(root);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index a37622624b..d2730b342c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -49,7 +49,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
@@ -145,7 +144,11 @@ class DynamoDbStreamsEventSourcePollerTest {
private DynamoDbStreamsEventSourcePoller pollerOver(DynamoDbStreamReader streamReader, EsmStore store) {
return new DynamoDbStreamsEventSourcePoller(
mock(Vertx.class), streamReader, executorService, new LambdaTargetResolver(functionStore, aliasStore),
- store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
+ store, OBJECT_MAPPER, config, filterMatcher, batchFailures(), clock::get);
+ }
+
+ private StreamBatchFailures batchFailures() {
+ return new StreamBatchFailures(sqsService, snsService, s3Service, OBJECT_MAPPER, config);
}
/**
@@ -243,7 +246,7 @@ class DynamoDbStreamsEventSourcePollerTest {
Vertx vertx = mock(Vertx.class);
DynamoDbStreamsEventSourcePoller p = new DynamoDbStreamsEventSourcePoller(
vertx, reader, executorService, new LambdaTargetResolver(functionStore, aliasStore),
- esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);
+ esmStore, OBJECT_MAPPER, config, filterMatcher, batchFailures(), clock::get);
p.startPersistedPollers();
@@ -988,17 +991,6 @@ class DynamoDbStreamsEventSourcePollerTest {
S3Service s3Service, AccountAwareStorageBackend<S3Object> objects) {
}
- @Test
- void s3OnFailureKeyUsesUtcLayout() {
- UUID randomId = UUID.fromString("12345678-1234-1234-1234-123456789abc");
-
- String key = DynamoDbStreamsEventSourcePoller.buildS3OnFailureKey(
- "esm-1", "shard-1", Instant.parse("2026-09-14T23:07:08Z"), randomId);
-
- assertEquals("aws/lambda/esm-1/shard-1/2026/09/14/"
- + "2026-09-14T23.07.08-12345678-1234-1234-1234-123456789abc", key);
- }
-
@Test
void onFailurePayloadUsesEpochSecondsNotMillisForArrivalTimestamps() throws Exception {
// Chosen so the seconds/millis interpretations land 46+ years apart: a millis bug cannot
@@ -1029,13 +1021,10 @@ class DynamoDbStreamsEventSourcePollerTest {
ArgumentCaptor<String> bodyCaptor = ArgumentCaptor.forClass(String.class);
verify(sqsService, timeout(2000)).sendMessage(anyString(), bodyCaptor.capture(), anyInt(), anyString());
- JsonNode dlqPayload = OBJECT_MAPPER.readTree(bodyCaptor.getValue());
- Instant first = Instant.parse(
- dlqPayload.path("DDBStreamBatchInfo").path("approximateArrivalOfFirstRecord").asText());
- Instant last = Instant.parse(
- dlqPayload.path("DDBStreamBatchInfo").path("approximateArrivalOfLastRecord").asText());
- assertEquals(knownEpochSeconds, first.getEpochSecond());
- assertEquals(knownEpochSeconds, last.getEpochSecond());
+ // 1_700_000_000 epoch seconds, written in whole seconds as in the AWS DynamoDB Streams example.
+ JsonNode batchInfo = OBJECT_MAPPER.readTree(bodyCaptor.getValue()).path("DDBStreamBatchInfo");
+ assertEquals("2023-11-14T22:13:20Z", batchInfo.path("approximateArrivalOfFirstRecord").asText());
+ assertEquals("2023-11-14T22:13:20Z", batchInfo.path("approximateArrivalOfLastRecord").asText());
}
@Test
@@ -1323,7 +1312,7 @@ class DynamoDbStreamsEventSourcePollerTest {
void retryBackoffDoublesFromThePollIntervalAndCapsOut() {
assertEquals(2_000, poller.retryBackoffMs(1));
assertEquals(4_000, poller.retryBackoffMs(2));
- assertEquals(DynamoDbStreamsEventSourcePoller.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));
+ assertEquals(StreamBatchFailures.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java
index c498d20cbd..17eaf41686 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java
@@ -1,12 +1,9 @@
package io.github.hectorvent.floci.services.lambda;
import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.DockerClient;
-import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.services.sqs.SqsService;
-import io.github.hectorvent.floci.services.sqs.model.Message;
-import io.github.hectorvent.floci.services.sqs.model.Queue;
+import io.github.hectorvent.floci.testing.SqsTestQueue;
import io.quarkus.test.junit.QuarkusTest;
import jakarta.inject.Inject;
import org.junit.jupiter.api.Assumptions;
@@ -17,8 +14,6 @@ import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Base64;
-import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -26,7 +21,6 @@ import java.util.zip.ZipOutputStream;
import static io.restassured.RestAssured.given;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
/**
* Runs a real failing function to prove the asynchronous retries wait as they do on AWS (scaled down by
@@ -41,7 +35,6 @@ class LambdaAsyncRetryDockerIntegrationTest {
private static final String ACCOUNT = "000000000000";
private static final String ROLE = "arn:aws:iam::" + ACCOUNT + ":role/lambda-role";
private static final Duration RECEIVE_DEADLINE = Duration.ofSeconds(60);
- private static final ObjectMapper MAPPER = new ObjectMapper();
@Inject
DockerClient dockerClient;
@@ -68,7 +61,7 @@ class LambdaAsyncRetryDockerIntegrationTest {
.then()
.statusCode(202);
- JsonNode record = awaitRecord(fixture.aliasQueueUrl());
+ JsonNode record = fixture.aliasQueue().awaitJson(RECEIVE_DEADLINE);
long elapsedMillis = Duration.ofNanos(System.nanoTime() - started).toMillis();
assertTrue(elapsedMillis >= 3000,
@@ -79,7 +72,7 @@ class LambdaAsyncRetryDockerIntegrationTest {
assertTrue(functionArn.endsWith(":1"), "functionArn should name the executed version: " + functionArn);
assertEquals("1", record.path("responseContext").path("executedVersion").asText());
assertEquals("alias", record.path("requestPayload").path("probe").asText());
- assertTrue(receiveOnce(fixture.latestQueueUrl()).isEmpty(),
+ assertTrue(fixture.latestQueue().receive().isEmpty(),
"the unqualified OnFailure destination should receive nothing");
} finally {
deleteFunction(fixture.functionName());
@@ -98,12 +91,12 @@ class LambdaAsyncRetryDockerIntegrationTest {
.then()
.statusCode(202);
- JsonNode record = awaitRecord(fixture.latestQueueUrl());
+ JsonNode record = fixture.latestQueue().awaitJson(RECEIVE_DEADLINE);
assertEquals("RetriesExhausted", record.path("requestContext").path("condition").asText());
assertEquals(1, record.path("requestContext").path("approximateInvokeCount").asInt());
assertEquals("latest", record.path("requestPayload").path("probe").asText());
- assertTrue(receiveOnce(fixture.aliasQueueUrl()).isEmpty(),
+ assertTrue(fixture.aliasQueue().receive().isEmpty(),
"the alias OnFailure destination should receive nothing");
} finally {
deleteFunction(fixture.functionName());
@@ -113,8 +106,8 @@ class LambdaAsyncRetryDockerIntegrationTest {
private Fixture setUp() throws Exception {
String suffix = UUID.randomUUID().toString().substring(0, 8);
String functionName = "async-retry-" + suffix;
- String latestQueueUrl = createQueue("async-retry-latest-" + suffix);
- String aliasQueueUrl = createQueue("async-retry-alias-" + suffix);
+ SqsTestQueue latestQueue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "async-retry-latest-");
+ SqsTestQueue aliasQueue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "async-retry-alias-");
given()
.contentType("application/json")
@@ -152,7 +145,7 @@ class LambdaAsyncRetryDockerIntegrationTest {
given()
.contentType("application/json")
.body("{\"MaximumRetryAttempts\":0,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"
- .formatted(queueArn(latestQueueUrl)))
+ .formatted(latestQueue.arn()))
.when()
.put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config")
.then()
@@ -161,39 +154,13 @@ class LambdaAsyncRetryDockerIntegrationTest {
given()
.contentType("application/json")
.body("{\"MaximumRetryAttempts\":2,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"
- .formatted(queueArn(aliasQueueUrl)))
+ .formatted(aliasQueue.arn()))
.when()
.put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config?Qualifier=live")
.then()
.statusCode(200);
- return new Fixture(functionName, latestQueueUrl, aliasQueueUrl);
- }
-
- private String createQueue(String queueName) {
- Queue queue = RequestScopes.callAs(ACCOUNT, () -> sqsService.createQueue(queueName, Map.of(), REGION));
- return queue.getQueueUrl();
- }
-
- private String queueArn(String queueUrl) {
- return RequestScopes.callAs(ACCOUNT, () ->
- sqsService.getQueueAttributes(queueUrl, List.of("QueueArn"), REGION).get("QueueArn"));
- }
-
- private List<Message> receiveOnce(String queueUrl) {
- return RequestScopes.callAs(ACCOUNT, () -> sqsService.receiveMessage(queueUrl, 1, 30, 0, REGION));
- }
-
- private JsonNode awaitRecord(String queueUrl) throws Exception {
- long deadline = System.nanoTime() + RECEIVE_DEADLINE.toNanos();
- while (System.nanoTime() < deadline) {
- List<Message> messages = receiveOnce(queueUrl);
- if (!messages.isEmpty()) {
- return MAPPER.readTree(messages.get(0).getBody());
- }
- Thread.sleep(250);
- }
- return fail("no OnFailure record reached " + queueUrl + " within " + RECEIVE_DEADLINE.toSeconds() + " s");
+ return new Fixture(functionName, latestQueue, aliasQueue);
}
private static void deleteFunction(String functionName) {
@@ -221,6 +188,6 @@ class LambdaAsyncRetryDockerIntegrationTest {
}
}
- private record Fixture(String functionName, String latestQueueUrl, String aliasQueueUrl) {
+ private record Fixture(String functionName, SqsTestQueue latestQueue, SqsTestQueue aliasQueue) {
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.java
new file mode 100644
index 0000000000..2177368627
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.java
@@ -0,0 +1,83 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.testing.SqsTestQueue;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * A poller thread carries no request context, so account-aware stores fall back to the default
+ * account. A mapping outside it must still reach its SQS and SNS destinations, as the poller sends
+ * from such a thread. The test thread has no request context either.
+ */
+@QuarkusTest
+class StreamBatchFailuresAccountIntegrationTest {
+
+ private static final String ACCOUNT = "111122223333";
+ private static final String REGION = "us-east-1";
+ private static final Duration DELIVERY_TIMEOUT = Duration.ofSeconds(5);
+ private static final StreamBatchFailures.DiscardedBatch BATCH = new StreamBatchFailures.DiscardedBatch(
+ "DDBStreamBatchInfo", "shardId-000000000000", "s1", "s1",
+ "2026-10-20T12:07:26Z", "2026-10-20T12:07:26Z", 1,
+ "arn:aws:dynamodb:us-east-1:111122223333:table/t/stream/2026-08-01T00:00:00.000", () -> "{\"Records\":[]}");
+
+ @Inject
+ StreamBatchFailures failures;
+
+ @Inject
+ SqsService sqsService;
+
+ @Inject
+ SnsService snsService;
+
+ @Test
+ void sqsDestinationOutsideTheDefaultAccountGetsTheRecord() throws Exception {
+ SqsTestQueue queue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "esm-failures-");
+
+ failures.sendToOnFailureDestination(mappingTo(queue.arn()), BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ assertEquals("RetryAttemptsExhausted",
+ queue.awaitJson(DELIVERY_TIMEOUT).at("/requestContext/condition").asText());
+ }
+
+ @Test
+ void snsDestinationOutsideTheDefaultAccountGetsTheRecord() throws Exception {
+ SqsTestQueue queue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "esm-failures-");
+ String topicArn = RequestScopes.callAs(ACCOUNT, () -> snsService.createTopic(
+ "esm-failures-" + UUID.randomUUID().toString().substring(0, 8), Map.of(), null, REGION).getTopicArn());
+ RequestScopes.runAs(ACCOUNT, () -> snsService.subscribe(topicArn, "sqs", queue.arn(), REGION,
+ Map.of("RawMessageDelivery", "true")));
+
+ failures.sendToOnFailureDestination(mappingTo(topicArn), BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ assertEquals("RetryAttemptsExhausted",
+ queue.awaitJson(DELIVERY_TIMEOUT).at("/requestContext/condition").asText());
+ }
+
+ private static EventSourceMapping mappingTo(String destinationArn) {
+ EventSourceMapping esm = new EventSourceMapping();
+ esm.setUuid("esm-1");
+ esm.setAccountId(ACCOUNT);
+ esm.setRegion(REGION);
+ esm.setEventSourceArn(BATCH.streamArn());
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination(destinationArn);
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ esm.setDestinationConfig(destinationConfig);
+ return esm;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.java
new file mode 100644
index 0000000000..f4db37e5bd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.java
@@ -0,0 +1,236 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.sns.SnsService;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.testutil.LogCapture;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+class StreamBatchFailuresTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final String STREAM_ARN = "arn:aws:dynamodb:us-east-1:000000000000:table/t/stream/2026-08-01T00:00:00.000";
+ private static final String EVENT = "{\"Records\":[{\"eventSource\":\"aws:dynamodb\"}]}";
+ /** Whole seconds: {@code ISO_INSTANT} would drop the fraction that AWS always writes. */
+ private static final StreamBatchFailures.DiscardedBatch BATCH = batch(() -> EVENT);
+
+ private SqsService sqsService;
+ private SnsService snsService;
+ private S3Service s3Service;
+ private StreamBatchFailures failures;
+
+ @BeforeEach
+ void setUp() {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ sqsService = mock(SqsService.class);
+ snsService = mock(SnsService.class);
+ s3Service = mock(S3Service.class);
+ failures = new StreamBatchFailures(sqsService, snsService, s3Service, MAPPER, config);
+ }
+
+ @Test
+ void s3KeyUsesTheUtcLayout() {
+ UUID randomId = UUID.fromString("12345678-1234-1234-1234-123456789abc");
+
+ String key = StreamBatchFailures.s3Key("esm-1", "shard-1", Instant.parse("2026-09-14T23:07:08Z"), randomId);
+
+ assertEquals("aws/lambda/esm-1/shard-1/2026/09/14/"
+ + "2026-09-14T23.07.08-12345678-1234-1234-1234-123456789abc", key);
+ }
+
+ @Test
+ void retryBackoffDoublesFromThePollIntervalAndCapsAtOneMinute() {
+ assertEquals(1_000, StreamBatchFailures.retryBackoffMs(500, 1));
+ assertEquals(2_000, StreamBatchFailures.retryBackoffMs(500, 2));
+ assertEquals(32_000, StreamBatchFailures.retryBackoffMs(500, 6));
+ assertEquals(60_000, StreamBatchFailures.retryBackoffMs(500, 7));
+ assertEquals(60_000, StreamBatchFailures.retryBackoffMs(500, 10_000), "a large retry count must not overflow");
+ }
+
+ @Test
+ void retryAttemptsRunOutAfterOnePlusTheMaximumInvocations() {
+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(null, null), 10_001), "unset means no limit");
+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(-1, null), 10_001), "-1 means no limit");
+ assertTrue(StreamBatchFailures.retryAttemptsExhausted(esm(0, null), 1));
+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(10, null), 10));
+ assertTrue(StreamBatchFailures.retryAttemptsExhausted(esm(10, null), 11));
+ }
+
+ @Test
+ void maximumRecordAgeIsExceededOnlyForAFiniteLimit() {
+ Instant arrival = Instant.parse("2026-10-20T12:07:26.012Z");
+ long weekLater = arrival.plusSeconds(604_801).toEpochMilli();
+ long almostAMinute = arrival.toEpochMilli() + 59_999;
+ long aMinute = arrival.toEpochMilli() + 60_000;
+
+ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, null), arrival, weekLater), "unset");
+ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, -1), arrival, weekLater), "-1");
+ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, 60), arrival, almostAMinute));
+ assertTrue(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, 60), arrival, aMinute));
+ }
+
+ @Test
+ void sqsRecordFillsDefaultsWritesTheTimestampInMillisecondsAndDoesNotBuildTheEvent() throws Exception {
+ EventSourceMapping esm = esm(0, null);
+ esm.setDestinationConfig(destination("arn:aws:sqs:us-east-1:000000000000:dlq"));
+ StreamBatchFailures.DiscardedBatch batch = batch(() -> {
+ throw new AssertionError("only an S3 destination may build the event");
+ });
+
+ failures.sendToOnFailureDestination(esm, batch, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);
+ verify(sqsService, times(1)).sendMessage(eq("http://localhost:4566/000000000000/dlq"), body.capture(),
+ eq(0), eq("us-east-1"));
+ JsonNode record = MAPPER.readTree(body.getValue());
+ assertEquals("", record.at("/requestContext/requestId").asText(), record.toString());
+ assertEquals("", record.at("/requestContext/functionArn").asText());
+ assertEquals(List.of("statusCode", "executedVersion"), fieldNames(record.path("responseContext")),
+ "no functionError without a function error: " + record);
+ assertEquals(200, record.at("/responseContext/statusCode").asInt());
+ assertEquals("$LATEST", record.at("/responseContext/executedVersion").asText());
+ assertEquals("2023-11-14T22:13:20Z", record.at("/DDBStreamBatchInfo/approximateArrivalOfFirstRecord").asText(),
+ "the arrival times are written as the source gives them");
+ assertEquals("2023-11-14T22:13:21Z", record.at("/DDBStreamBatchInfo/approximateArrivalOfLastRecord").asText());
+ assertTrue(record.path("timestamp").asText().matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z"),
+ "timestamp must be ISO-8601 with exactly three fraction digits: " + record.path("timestamp"));
+ }
+
+ @Test
+ void destinationInAnotherRegionIsReachedInItsOwnRegion() {
+ EventSourceMapping esm = esm(0, null);
+ esm.setDestinationConfig(destination("arn:aws:sqs:eu-west-2:000000000000:dlq"));
+
+ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ verify(sqsService, times(1)).sendMessage(eq("http://localhost:4566/000000000000/dlq"), anyString(),
+ eq(0), eq("eu-west-2"));
+ }
+
+ @Test
+ void s3RecordCarriesTheEventAsAPayloadString() throws Exception {
+ EventSourceMapping esm = esm(0, null);
+ esm.setDestinationConfig(destination("arn:aws:s3:::failures"));
+ InvokeResult error = new InvokeResult(200, "Unhandled", new byte[0], null, "req-1");
+
+ failures.sendToOnFailureDestination(esm, BATCH, error, 1, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED);
+
+ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);
+ verify(s3Service, times(1)).putObject(eq("failures"), key.capture(), body.capture(),
+ eq("application/json"), eq(Map.of()));
+ assertTrue(key.getValue().startsWith("aws/lambda/esm-1/shardId-000000000000/"), key.getValue());
+ JsonNode record = MAPPER.readTree(new String(body.getValue(), StandardCharsets.UTF_8));
+ assertEquals(List.of("requestContext", "responseContext", "version", "payload", "timestamp", "DDBStreamBatchInfo"),
+ fieldNames(record));
+ assertEquals(EVENT, record.path("payload").asText());
+ assertEquals("MaximumRecordAgeExceeded", record.at("/requestContext/condition").asText());
+ assertEquals("Unhandled", record.at("/responseContext/functionError").asText());
+ verifyNoInteractions(sqsService, snsService);
+ }
+
+ @Test
+ void refusedSendIsLoggedWithItsCauseNotThrown() {
+ EventSourceMapping esm = esm(0, null);
+ esm.setDestinationConfig(destination("arn:aws:sqs:us-east-1:000000000000:gone"));
+ AwsException refusal = new AwsException("AWS.SimpleQueueService.NonExistentQueue",
+ "The specified queue does not exist.", 400);
+ when(sqsService.sendMessage(anyString(), anyString(), anyInt(), anyString())).thenThrow(refusal);
+
+ List<LogRecord> records = LogCapture.capture(StreamBatchFailures.class, () ->
+ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED));
+
+ LogRecord error = records.stream()
+ .filter(record -> record.getLevel().intValue() >= Level.SEVERE.intValue())
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("no error was logged: " + records));
+ assertSame(refusal, error.getThrown(), "the error log must keep the cause and its stack trace");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", "arn:aws:lambda:us-east-1:000000000000:function:other", "arn:aws:s3:::"})
+ void unusableDestinationSendsNothing(String destinationArn) {
+ EventSourceMapping esm = esm(0, null);
+ esm.setDestinationConfig(destination(destinationArn));
+
+ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ verifyNoInteractions(sqsService, snsService, s3Service);
+ }
+
+ @Test
+ void mappingWithoutDestinationSendsNothing() {
+ failures.sendToOnFailureDestination(esm(0, null), BATCH, new InvokeResult(), 1,
+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);
+
+ verifyNoInteractions(sqsService, snsService, s3Service);
+ }
+
+ private static StreamBatchFailures.DiscardedBatch batch(Supplier<String> event) {
+ return new StreamBatchFailures.DiscardedBatch("DDBStreamBatchInfo", "shardId-000000000000", "s1", "s2",
+ "2023-11-14T22:13:20Z", "2023-11-14T22:13:21Z", 2, STREAM_ARN, event);
+ }
+
+ private static EventSourceMapping esm(Integer maximumRetryAttempts, Integer maximumRecordAgeInSeconds) {
+ EventSourceMapping esm = new EventSourceMapping();
+ esm.setUuid("esm-1");
+ esm.setAccountId("000000000000");
+ esm.setRegion("us-east-1");
+ esm.setEventSourceArn(STREAM_ARN);
+ esm.setMaximumRetryAttempts(maximumRetryAttempts);
+ esm.setMaximumRecordAgeInSeconds(maximumRecordAgeInSeconds);
+ return esm;
+ }
+
+ private static EventSourceMapping.DestinationConfig destination(String arn) {
+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();
+ onFailure.setDestination(arn);
+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();
+ destinationConfig.setOnFailure(onFailure);
+ return destinationConfig;
+ }
+
+ private static List<String> fieldNames(JsonNode node) {
+ List<String> names = new ArrayList<>();
+ node.fieldNames().forEachRemaining(names::add);
+ return names;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.java b/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.java
new file mode 100644
index 0000000000..9a7497d151
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.java
@@ -0,0 +1,74 @@
+package io.github.hectorvent.floci.testing;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.sqs.SqsService;
+import io.github.hectorvent.floci.services.sqs.model.Message;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.awaitility.Awaitility.await;
+
+/**
+ * An SQS queue a test creates in one account and Region, typically as a destination, and then waits
+ * on for one JSON message. Every call runs in that account, so it works from a thread without a
+ * request context.
+ */
+public final class SqsTestQueue {
+
+ /** Longer than any test, so a received message never comes back and fails a later check. */
+ private static final int VISIBILITY_TIMEOUT_SECONDS = 300;
+ private static final Duration POLL_INTERVAL = Duration.ofMillis(250);
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private final SqsService sqsService;
+ private final String account;
+ private final String region;
+ private final String url;
+
+ private SqsTestQueue(SqsService sqsService, String account, String region, String url) {
+ this.sqsService = sqsService;
+ this.account = account;
+ this.region = region;
+ this.url = url;
+ }
+
+ /** Creates a queue named {@code namePrefix} plus a random suffix. */
+ public static SqsTestQueue create(SqsService sqsService, String account, String region, String namePrefix) {
+ String name = namePrefix + UUID.randomUUID().toString().substring(0, 8);
+ String url = RequestScopes.callAs(account, () -> sqsService.createQueue(name, Map.of(), region).getQueueUrl());
+ return new SqsTestQueue(sqsService, account, region, url);
+ }
+
+ public String url() {
+ return url;
+ }
+
+ public String arn() {
+ return RequestScopes.callAs(account, () ->
+ sqsService.getQueueAttributes(url, List.of("QueueArn"), region).get("QueueArn"));
+ }
+
+ /** At most one message, received without waiting. */
+ public List<Message> receive() {
+ return RequestScopes.callAs(account, () ->
+ sqsService.receiveMessage(url, 1, VISIBILITY_TIMEOUT_SECONDS, 0, region));
+ }
+
+ /** Polls until a message arrives, and returns its body parsed as JSON. */
+ public JsonNode awaitJson(Duration timeout) throws Exception {
+ List<Message> messages = await("a message in " + url)
+ .atMost(timeout)
+ .pollInterval(POLL_INTERVAL)
+ .until(this::receive, received -> !received.isEmpty());
+ return MAPPER.readTree(messages.get(0).getBody());
+ }
+
+ public void delete() {
+ RequestScopes.runAs(account, () -> sqsService.deleteQueue(url, region));
+ }
+}