Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): share the stream OnFailure delivery and send it in the destination's account (#6224)
Attila Kisb0a4d5064bd0
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 8f41eed54a..0dabf3889f 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -5,9 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;-import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;-import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;@@ -16,26 +14,21 @@ import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.pipes.PipesFilterMatcher;-import io.github.hectorvent.floci.services.s3.S3Service;-import io.github.hectorvent.floci.services.sns.SnsService;-import io.github.hectorvent.floci.services.sqs.SqsService;import io.vertx.core.Vertx;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;-import java.nio.charset.StandardCharsets;import java.time.Instant;-import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;import java.util.ArrayList;import java.util.HashMap;import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.OptionalLong;import java.util.Set;-import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -46,14 +39,10 @@ import java.util.stream.Collectors;public class DynamoDbStreamsEventSourcePoller implements Resettable {private static final Logger LOG = Logger.getLogger(DynamoDbStreamsEventSourcePoller.class);- private static final DateTimeFormatter S3_ON_FAILURE_PATH_DATE_FORMATTER =- DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);- private static final DateTimeFormatter S3_ON_FAILURE_FILE_TIME_FORMATTER =- DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH.mm.ss").withZone(ZoneOffset.UTC);+ private static final String BATCH_INFO_NAME = "DDBStreamBatchInfo";/** Raised by the stream when a stored checkpoint has aged out of the retained window. */private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";- static final long MAX_RETRY_BACKOFF_MS = 60_000;private final Vertx vertx;private final DynamoDbStreamReader streamReader;@@ -62,10 +51,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private final EsmStore esmStore;private final ObjectMapper objectMapper;private final PipesFilterMatcher filterMatcher;- private final SqsService sqsService;- private final SnsService snsService;- private final S3Service s3Service;- private final String baseUrl;+ private final StreamBatchFailures batchFailures;private final long pollIntervalMs;private final LongSupplier clock;private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();@@ -92,11 +78,9 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {ObjectMapper objectMapper,EmulatorConfig config,PipesFilterMatcher filterMatcher,- SqsService sqsService,- SnsService snsService,- S3Service s3Service) {+ StreamBatchFailures batchFailures) {this(vertx, streamReader, executorService, targetResolver, esmStore, objectMapper, config,- filterMatcher, sqsService, snsService, s3Service, System::currentTimeMillis);+ filterMatcher, batchFailures, System::currentTimeMillis);}DynamoDbStreamsEventSourcePoller(Vertx vertx, DynamoDbStreamReader streamReader,@@ -106,9 +90,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {ObjectMapper objectMapper,EmulatorConfig config,PipesFilterMatcher filterMatcher,- SqsService sqsService,- SnsService snsService,- S3Service s3Service,+ StreamBatchFailures batchFailures,LongSupplier clock) {this.vertx = vertx;this.streamReader = streamReader;@@ -117,11 +99,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {this.esmStore = esmStore;this.objectMapper = objectMapper;this.pollIntervalMs = config.services().lambda().pollIntervalMs();- this.baseUrl = config.effectiveBaseUrl();this.filterMatcher = filterMatcher;- this.sqsService = sqsService;- this.snsService = snsService;- this.s3Service = s3Service;+ this.batchFailures = batchFailures;this.clock = clock;}@@ -294,7 +273,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private void pollShard(EventSourceMapping esm, LambdaFunction fn, DynamoDbStreamReader.Stream stream,String shardId) {String lastSeq = esm.getShardSequenceNumbers().get(shardId);- String persistedKey = batchStateKey(esm.getUuid(), shardId, lastSeq);+ String persistedKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, lastSeq);boolean checkpointTrimmed = false;DynamoDbStreamReader.RecordsPage page;@@ -317,7 +296,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {+ "trim horizon; records between were dropped from the stream",esm.getUuid(), lastSeq);page = streamReader.readAfter(stream, shardId, null, Math.min(esm.getBatchSize(),- bisectLimit(esm, batchStateKey(esm.getUuid(), shardId, null))));+ bisectLimit(esm, StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, null))));}List<DynamoDbStreamReader.Record> records = page.records();@@ -353,7 +332,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {records, filterNodes, filterMatcher.applyFilterCriteria(filterNodes, filterParams));}- String batchKey = batchStateKey(esm.getUuid(), shardId, checkpointTrimmed ? null : lastSeq);+ String batchKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, checkpointTrimmed ? null : lastSeq);if (matched.isEmpty()) {clearBatchState(batchKey);advanceCheckpoint(esm, shardId, newestFetchedSeq);@@ -397,8 +376,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {clearBatchState(batchKey);String nextCheckpoint = outcome.checkpoint();- String nextBatchKey = batchStateKey(esm.getUuid(), shardId, nextCheckpoint);- Integer maxRetries = esm.getMaximumRetryAttempts();+ String nextBatchKey = StreamBatchFailures.batchStateKey(esm.getUuid(), shardId, nextCheckpoint);int currentRetries = retryCounts.merge(nextBatchKey, 1, Integer::sum);Set<String> deliveredSeqs = new HashSet<>();for (DynamoDbStreamReader.Record rec : matched) {@@ -419,12 +397,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",esm.getUuid(), newestFetchedSeq);dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));- } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {+ currentRetries, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED, newestFetchedSeq));+ } else if (StreamBatchFailures.retryAttemptsExhausted(esm, currentRetries)) {LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",- esm.getUuid(), maxRetries, newestFetchedSeq);+ esm.getUuid(), esm.getMaximumRetryAttempts(), newestFetchedSeq);dispose(esm, shardId, nextBatchKey, new PendingFailure(failedRecords, invokeResult,- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));+ currentRetries, StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED, newestFetchedSeq));} else {retryNotBefore.put(nextBatchKey, now + retryBackoffMs(currentRetries));LOG.warnv("DynamoDB Streams ESM {0}: Lambda returned error [batchItemFailures], retry {1}, records will be retried",@@ -432,14 +410,13 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {advanceCheckpoint(esm, shardId, nextCheckpoint);}} else {- Integer maxRetries = esm.getMaximumRetryAttempts();int currentRetries = retryCounts.getOrDefault(batchKey, 0) + 1;if (hasExceededMaximumRecordAge(esm, matched, now)) {LOG.warnv("DynamoDB Streams ESM {0}: maximum record age exceeded for batch ending at {1}",esm.getUuid(), newestFetchedSeq);clearBatchState(batchKey);dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,- currentRetries, "MaximumRecordAgeExceeded", newestFetchedSeq));+ currentRetries, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED, newestFetchedSeq));} else if (invokeResult.getFunctionError() != null&& Boolean.TRUE.equals(esm.getBisectBatchOnFunctionError()) && matched.size() > 1) {// ponytail: the halved limit is keyed by checkpoint, so once a good half advances the@@ -450,12 +427,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {bisectLimits.put(batchKey, limit);LOG.infov("DynamoDB Streams ESM {0}: function error on {1} record(s), bisecting to {2}",esm.getUuid(), records.size(), limit);- } else if (maxRetries != null && maxRetries >= 0 && currentRetries > maxRetries) {+ } else if (StreamBatchFailures.retryAttemptsExhausted(esm, currentRetries)) {LOG.warnv("DynamoDB Streams ESM {0}: maximum retry attempts ({1}) exhausted for batch ending at {2}",- esm.getUuid(), maxRetries, newestFetchedSeq);+ esm.getUuid(), esm.getMaximumRetryAttempts(), newestFetchedSeq);clearBatchState(batchKey);dispose(esm, shardId, persistedKey, new PendingFailure(matched, invokeResult,- currentRetries, "RetryAttemptsExhausted", newestFetchedSeq));+ currentRetries, StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED, newestFetchedSeq));} else {retryCounts.put(batchKey, currentRetries);retryNotBefore.put(batchKey, now + retryBackoffMs(currentRetries));@@ -479,8 +456,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {clearBatchState(key);return;}- sendToOnFailureDestination(esm, shardId, failure.records(), failure.invokeResult(),- failure.invokeCount(), failure.condition());+ batchFailures.sendToOnFailureDestination(esm, discardedBatch(esm, shardId, failure.records()),+ failure.invokeResult(), failure.invokeCount(), failure.condition());clearBatchState(key);advanceCheckpoint(esm, shardId, failure.advanceTo());}@@ -501,10 +478,6 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {return uuid + ":" + shardId;}- private static String batchStateKey(String uuid, String shardId, String sequence) {- return uuid + ":" + shardId + ":" + (sequence == null ? "TRIM_HORIZON" : sequence);- }-/*** Returns the last record that can be consumed after a successful invocation. Floci stores the* last consumed sequence and resumes with {@code AFTER_SEQUENCE_NUMBER}, so a partial failure@@ -580,20 +553,12 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private boolean hasExceededMaximumRecordAge(EventSourceMapping esm, List<DynamoDbStreamReader.Record> records,long now) {- Integer configuredAge = esm.getMaximumRecordAgeInSeconds();- if (configuredAge == null || configuredAge <= 0) {- return false;- }-- long oldestRecordSeconds = records.stream()+ OptionalLong oldestRecordSeconds = records.stream().mapToLong(DynamoDbStreamsEventSourcePoller::approximateCreationSeconds).filter(seconds -> seconds > 0)- .min()- .orElse(0);- if (oldestRecordSeconds == 0) {- return false;- }- return now - oldestRecordSeconds * 1_000 >= configuredAge * 1_000L;+ .min();+ return oldestRecordSeconds.isPresent() && StreamBatchFailures.maximumRecordAgeExceeded(+ esm, Instant.ofEpochSecond(oldestRecordSeconds.getAsLong()), now);}private static long approximateCreationSeconds(DynamoDbStreamReader.Record rec) {@@ -601,125 +566,22 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}long retryBackoffMs(int retries) {- int doublings = Math.min(retries, 16);- return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);+ return StreamBatchFailures.retryBackoffMs(pollIntervalMs, retries);}- private void sendToOnFailureDestination(EventSourceMapping esm, String shardId,- List<DynamoDbStreamReader.Record> records,- InvokeResult invokeResult,- int invokeCount,- String condition) {- if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {- return;- }- String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();- if (destinationArn == null || destinationArn.isBlank()) {- return;- }-- try {- if (destinationArn.contains(":sqs:")) {- String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());- String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);- sqsService.sendMessage(queueUrl, payload, 0, region);- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SQS DLQ {1}", esm.getUuid(), destinationArn);- } else if (destinationArn.contains(":sns:")) {- String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);- snsService.publish(destinationArn, null, payload, "ESM OnFailure", region);- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SNS DLQ {1}", esm.getUuid(), destinationArn);- } else if (destinationArn.contains(":s3:")) {- AwsArnUtils.Arn arn = AwsArnUtils.parse(destinationArn);- if (!"s3".equals(arn.service()) || arn.resource().isBlank()) {- throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);- }- String key = buildS3OnFailureKey(esm.getUuid(), shardId, Instant.now(), UUID.randomUUID());- String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition);- RequestScopes.runAs(esm.getAccountId(), () ->- s3Service.putObject(arn.resource(), key,- s3Payload.getBytes(StandardCharsets.UTF_8),- "application/json", Map.of()));- LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to S3 bucket {1} with key {2}",- esm.getUuid(), arn.resource(), key);- } else {- LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}; discarding records",- esm.getUuid(), destinationArn);- }- } catch (Exception e) {- LOG.errorv("DynamoDB Streams ESM {0}: failed to send to OnFailure destination {1}; discarding records: {2}",- esm.getUuid(), destinationArn, e.getMessage());- }+ private StreamBatchFailures.DiscardedBatch discardedBatch(EventSourceMapping esm, String shardId,+ List<DynamoDbStreamReader.Record> records) {+ DynamoDbStreamReader.Record first = records.get(0);+ DynamoDbStreamReader.Record last = records.get(records.size() - 1);+ return new StreamBatchFailures.DiscardedBatch(BATCH_INFO_NAME, shardId,+ first.sequenceNumber(), last.sequenceNumber(),+ creationTime(first), creationTime(last),+ records.size(), esm.getEventSourceArn(), () -> buildDynamoDbEvent(records, esm));}- private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,- List<DynamoDbStreamReader.Record> records,- InvokeResult invokeResult,- int invokeCount,- String condition) {- try {- ObjectNode root = (ObjectNode) objectMapper.readTree(- buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount, condition));- root.put("payload", buildDynamoDbEvent(records, esm));- return objectMapper.writeValueAsString(root);- } catch (Exception e) {- throw new IllegalStateException("Failed to serialize S3 OnFailure payload", e);- }- }-- static String buildS3OnFailureKey(String esmUuid, String shardId, Instant now, UUID randomId) {- return "aws/lambda/" + esmUuid + "/" + shardId + "/"- + S3_ON_FAILURE_PATH_DATE_FORMATTER.format(now) + "/"- + S3_ON_FAILURE_FILE_TIME_FORMATTER.format(now) + "-" + randomId;- }-- private String buildOnFailurePayload(EventSourceMapping esm, String shardId,- List<DynamoDbStreamReader.Record> records,- InvokeResult invokeResult,- int invokeCount,- String condition) {- try {- ObjectNode root = objectMapper.createObjectNode();- root.put("version", "1.0");- root.put("timestamp", DateTimeFormatter.ISO_INSTANT.format(Instant.now()));-- ObjectNode requestContext = root.putObject("requestContext");- requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");- requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");- requestContext.put("condition", condition);- requestContext.put("approximateInvokeCount", invokeCount);-- ObjectNode responseContext = root.putObject("responseContext");- responseContext.put("statusCode", invokeResult.getStatusCode() != 0 ? invokeResult.getStatusCode() : 200);- responseContext.put("executedVersion", invokeResult.getExecutedVersion() != null ? invokeResult.getExecutedVersion() : "$LATEST");- if (invokeResult.getFunctionError() != null) {- responseContext.put("functionError", invokeResult.getFunctionError());- }-- ObjectNode batchInfo = root.putObject("DDBStreamBatchInfo");- batchInfo.put("shardId", shardId);- String startSeq = records.isEmpty() ? "" : records.get(0).sequenceNumber();- String endSeq = records.isEmpty() ? "" : records.get(records.size() - 1).sequenceNumber();- batchInfo.put("startSequenceNumber", startSeq);- batchInfo.put("endSequenceNumber", endSeq);-- if (!records.isEmpty()) {- long firstArrival = approximateCreationSeconds(records.get(0));- long lastArrival = approximateCreationSeconds(records.get(records.size() - 1));- batchInfo.put("approximateArrivalOfFirstRecord",- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(firstArrival)));- batchInfo.put("approximateArrivalOfLastRecord",- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(lastArrival)));- }- batchInfo.put("batchSize", records.size());- batchInfo.put("streamArn", esm.getEventSourceArn());-- return objectMapper.writeValueAsString(root);- } catch (Exception e) {- LOG.warnv("Failed to serialize OnFailure payload: {0}", e.getMessage());- return "{}";- }+ /** A stream record's creation time is whole seconds, and AWS writes it without a fraction. */+ private static String creationTime(DynamoDbStreamReader.Record rec) {+ return DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochSecond(approximateCreationSeconds(rec)));}private String buildDynamoDbEvent(List<DynamoDbStreamReader.Record> records, EventSourceMapping esm) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.java b/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.javanew file mode 100644index 0000000000..2f08e4b7f9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailures.java@@ -0,0 +1,188 @@+package io.github.hectorvent.floci.services.lambda;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.Map;+import java.util.UUID;+import java.util.function.Supplier;++/**+ * What Lambda does with a stream batch that keeps failing: the retry backoff, the+ * {@code MaximumRetryAttempts} and {@code MaximumRecordAgeInSeconds} limits, and the invocation+ * record it sends to the mapping's {@code DestinationConfig.OnFailure} when it discards the batch.+ * Only the batch info block and the event depend on the stream source; its poller passes them in a+ * {@link DiscardedBatch}.+ */+@ApplicationScoped+public class StreamBatchFailures {++ private static final Logger LOG = Logger.getLogger(StreamBatchFailures.class);++ static final String RETRY_ATTEMPTS_EXHAUSTED = "RetryAttemptsExhausted";+ static final String MAXIMUM_RECORD_AGE_EXCEEDED = "MaximumRecordAgeExceeded";+ static final long MAX_RETRY_BACKOFF_MS = 60_000;++ private static final DateTimeFormatter S3_KEY_DATE =+ DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);+ private static final DateTimeFormatter S3_KEY_TIME =+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH.mm.ss").withZone(ZoneOffset.UTC);++ /**+ * The source-specific part of the invocation record. {@code batchInfoName} names the source's+ * batch info block, such as {@code DDBStreamBatchInfo}. The arrival times are written as given,+ * because their precision depends on the source. {@code event} builds the event of the discarded+ * records, in the shape the function gets it; only an S3 destination asks for it, as the+ * {@code payload} string.+ */+ record DiscardedBatch(String batchInfoName, String shardId, String startSequenceNumber,+ String endSequenceNumber, String firstArrival, String lastArrival,+ int batchSize, String streamArn, Supplier<String> event) {+ }++ private final SqsService sqsService;+ private final SnsService snsService;+ private final S3Service s3Service;+ private final ObjectMapper objectMapper;+ private final String baseUrl;++ @Inject+ public StreamBatchFailures(SqsService sqsService, SnsService snsService, S3Service s3Service,+ ObjectMapper objectMapper, EmulatorConfig config) {+ this.sqsService = sqsService;+ this.snsService = snsService;+ this.s3Service = s3Service;+ this.objectMapper = objectMapper;+ this.baseUrl = config.effectiveBaseUrl();+ }++ static long retryBackoffMs(long pollIntervalMs, int retries) {+ int doublings = Math.min(retries, 16);+ return Math.min(pollIntervalMs * (1L << doublings), MAX_RETRY_BACKOFF_MS);+ }++ /** {@code invokeCount} is 1 + the retries so far. An unset or {@code -1} limit never runs out. */+ static boolean retryAttemptsExhausted(EventSourceMapping esm, int invokeCount) {+ Integer maximum = esm.getMaximumRetryAttempts();+ return maximum != null && maximum >= 0 && invokeCount > maximum;+ }++ /** An unset or {@code -1} limit never expires a batch. */+ static boolean maximumRecordAgeExceeded(EventSourceMapping esm, Instant oldestArrival, long nowMillis) {+ Integer maximum = esm.getMaximumRecordAgeInSeconds();+ return maximum != null && maximum > 0 && nowMillis - oldestArrival.toEpochMilli() >= maximum * 1_000L;+ }++ /** The retry state of a failing batch is keyed by the checkpoint it is read after. */+ static String batchStateKey(String esmUuid, String shardId, String checkpoint) {+ return esmUuid + ":" + shardId + ":" + (checkpoint == null ? "TRIM_HORIZON" : checkpoint);+ }++ static String s3Key(String esmUuid, String shardId, Instant now, UUID randomId) {+ return "aws/lambda/" + esmUuid + "/" + shardId + "/"+ + S3_KEY_DATE.format(now) + "/"+ + S3_KEY_TIME.format(now) + "-" + randomId;+ }++ /**+ * Sends the invocation record of a discarded batch to the mapping's OnFailure destination, if it+ * has one. A refused send is logged and the batch is dropped all the same, as AWS does.+ */+ void sendToOnFailureDestination(EventSourceMapping esm, DiscardedBatch batch, InvokeResult invokeResult,+ int invokeCount, String condition) {+ if (esm.getDestinationConfig() == null || esm.getDestinationConfig().getOnFailure() == null) {+ return;+ }+ String destinationArn = esm.getDestinationConfig().getOnFailure().getDestination();+ if (destinationArn == null || destinationArn.isBlank()) {+ return;+ }++ // A poller thread carries no request context, so the destination's account is set before+ // its account-aware store is read.+ String destinationAccount = AwsArnUtils.accountOrDefault(destinationArn, esm.getAccountId());+ try {+ if (destinationArn.contains(":sqs:")) {+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());+ String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);+ String record = invocationRecord(esm, batch, invokeResult, invokeCount, condition, false);+ RequestScopes.runAs(destinationAccount, () -> sqsService.sendMessage(queueUrl, record, 0, region));+ } else if (destinationArn.contains(":sns:")) {+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());+ String record = invocationRecord(esm, batch, invokeResult, invokeCount, condition, false);+ RequestScopes.runAs(destinationAccount, () ->+ snsService.publish(destinationArn, null, record, "ESM OnFailure", region));+ } else if (destinationArn.contains(":s3:")) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(destinationArn);+ if (!"s3".equals(arn.service()) || arn.resource().isBlank()) {+ throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);+ }+ String key = s3Key(esm.getUuid(), batch.shardId(), Instant.now(), UUID.randomUUID());+ byte[] body = invocationRecord(esm, batch, invokeResult, invokeCount, condition, true)+ .getBytes(StandardCharsets.UTF_8);+ RequestScopes.runAs(esm.getAccountId(), () ->+ s3Service.putObject(arn.resource(), key, body, "application/json", Map.of()));+ } else {+ LOG.warnv("ESM {0}: unsupported OnFailure destination {1}, discarding the batch",+ esm.getUuid(), destinationArn);+ return;+ }+ LOG.infov("ESM {0}: sent the discarded batch of shard {1} to {2}",+ esm.getUuid(), batch.shardId(), destinationArn);+ } catch (Exception e) {+ LOG.errorv(e, "ESM {0}: failed to send to OnFailure destination {1}, discarding the batch",+ esm.getUuid(), destinationArn);+ }+ }++ private String invocationRecord(EventSourceMapping esm, DiscardedBatch batch, InvokeResult invokeResult,+ int invokeCount, String condition, boolean withPayload)+ throws JsonProcessingException {+ ObjectNode root = objectMapper.createObjectNode();+ ObjectNode requestContext = root.putObject("requestContext");+ requestContext.put("requestId", invokeResult.getRequestId() != null ? invokeResult.getRequestId() : "");+ requestContext.put("functionArn", esm.getFunctionArn() != null ? esm.getFunctionArn() : "");+ requestContext.put("condition", condition);+ requestContext.put("approximateInvokeCount", invokeCount);++ ObjectNode responseContext = root.putObject("responseContext");+ responseContext.put("statusCode", invokeResult.getStatusCode() != 0 ? invokeResult.getStatusCode() : 200);+ responseContext.put("executedVersion",+ invokeResult.getExecutedVersion() != null ? invokeResult.getExecutedVersion() : "$LATEST");+ if (invokeResult.getFunctionError() != null) {+ responseContext.put("functionError", invokeResult.getFunctionError());+ }++ root.put("version", "1.0");+ if (withPayload) {+ root.put("payload", batch.event().get());+ }+ root.put("timestamp", AsyncInvokeDestinationRouter.formatRecordTimestamp(Instant.now()));++ ObjectNode batchInfo = root.putObject(batch.batchInfoName());+ batchInfo.put("shardId", batch.shardId());+ batchInfo.put("startSequenceNumber", batch.startSequenceNumber());+ batchInfo.put("endSequenceNumber", batch.endSequenceNumber());+ batchInfo.put("approximateArrivalOfFirstRecord", batch.firstArrival());+ batchInfo.put("approximateArrivalOfLastRecord", batch.lastArrival());+ batchInfo.put("batchSize", batch.batchSize());+ batchInfo.put("streamArn", batch.streamArn());+ return objectMapper.writeValueAsString(root);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex a37622624b..d2730b342c 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -49,7 +49,6 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;-import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.CountDownLatch;@@ -145,7 +144,11 @@ class DynamoDbStreamsEventSourcePollerTest {private DynamoDbStreamsEventSourcePoller pollerOver(DynamoDbStreamReader streamReader, EsmStore store) {return new DynamoDbStreamsEventSourcePoller(mock(Vertx.class), streamReader, executorService, new LambdaTargetResolver(functionStore, aliasStore),- store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);+ store, OBJECT_MAPPER, config, filterMatcher, batchFailures(), clock::get);+ }++ private StreamBatchFailures batchFailures() {+ return new StreamBatchFailures(sqsService, snsService, s3Service, OBJECT_MAPPER, config);}/**@@ -243,7 +246,7 @@ class DynamoDbStreamsEventSourcePollerTest {Vertx vertx = mock(Vertx.class);DynamoDbStreamsEventSourcePoller p = new DynamoDbStreamsEventSourcePoller(vertx, reader, executorService, new LambdaTargetResolver(functionStore, aliasStore),- esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service, clock::get);+ esmStore, OBJECT_MAPPER, config, filterMatcher, batchFailures(), clock::get);p.startPersistedPollers();@@ -988,17 +991,6 @@ class DynamoDbStreamsEventSourcePollerTest {S3Service s3Service, AccountAwareStorageBackend<S3Object> objects) {}- @Test- void s3OnFailureKeyUsesUtcLayout() {- UUID randomId = UUID.fromString("12345678-1234-1234-1234-123456789abc");-- String key = DynamoDbStreamsEventSourcePoller.buildS3OnFailureKey(- "esm-1", "shard-1", Instant.parse("2026-09-14T23:07:08Z"), randomId);-- assertEquals("aws/lambda/esm-1/shard-1/2026/09/14/"- + "2026-09-14T23.07.08-12345678-1234-1234-1234-123456789abc", key);- }-@Testvoid onFailurePayloadUsesEpochSecondsNotMillisForArrivalTimestamps() throws Exception {// Chosen so the seconds/millis interpretations land 46+ years apart: a millis bug cannot@@ -1029,13 +1021,10 @@ class DynamoDbStreamsEventSourcePollerTest {ArgumentCaptor<String> bodyCaptor = ArgumentCaptor.forClass(String.class);verify(sqsService, timeout(2000)).sendMessage(anyString(), bodyCaptor.capture(), anyInt(), anyString());- JsonNode dlqPayload = OBJECT_MAPPER.readTree(bodyCaptor.getValue());- Instant first = Instant.parse(- dlqPayload.path("DDBStreamBatchInfo").path("approximateArrivalOfFirstRecord").asText());- Instant last = Instant.parse(- dlqPayload.path("DDBStreamBatchInfo").path("approximateArrivalOfLastRecord").asText());- assertEquals(knownEpochSeconds, first.getEpochSecond());- assertEquals(knownEpochSeconds, last.getEpochSecond());+ // 1_700_000_000 epoch seconds, written in whole seconds as in the AWS DynamoDB Streams example.+ JsonNode batchInfo = OBJECT_MAPPER.readTree(bodyCaptor.getValue()).path("DDBStreamBatchInfo");+ assertEquals("2023-11-14T22:13:20Z", batchInfo.path("approximateArrivalOfFirstRecord").asText());+ assertEquals("2023-11-14T22:13:20Z", batchInfo.path("approximateArrivalOfLastRecord").asText());}@Test@@ -1323,7 +1312,7 @@ class DynamoDbStreamsEventSourcePollerTest {void retryBackoffDoublesFromThePollIntervalAndCapsOut() {assertEquals(2_000, poller.retryBackoffMs(1));assertEquals(4_000, poller.retryBackoffMs(2));- assertEquals(DynamoDbStreamsEventSourcePoller.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));+ assertEquals(StreamBatchFailures.MAX_RETRY_BACKOFF_MS, poller.retryBackoffMs(20));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.javaindex c498d20cbd..17eaf41686 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaAsyncRetryDockerIntegrationTest.java@@ -1,12 +1,9 @@package io.github.hectorvent.floci.services.lambda;import com.fasterxml.jackson.databind.JsonNode;-import com.fasterxml.jackson.databind.ObjectMapper;import com.github.dockerjava.api.DockerClient;-import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.services.sqs.SqsService;-import io.github.hectorvent.floci.services.sqs.model.Message;-import io.github.hectorvent.floci.services.sqs.model.Queue;+import io.github.hectorvent.floci.testing.SqsTestQueue;import io.quarkus.test.junit.QuarkusTest;import jakarta.inject.Inject;import org.junit.jupiter.api.Assumptions;@@ -17,8 +14,6 @@ import java.io.ByteArrayOutputStream;import java.nio.charset.StandardCharsets;import java.time.Duration;import java.util.Base64;-import java.util.List;-import java.util.Map;import java.util.UUID;import java.util.zip.ZipEntry;import java.util.zip.ZipOutputStream;@@ -26,7 +21,6 @@ import java.util.zip.ZipOutputStream;import static io.restassured.RestAssured.given;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;-import static org.junit.jupiter.api.Assertions.fail;/*** Runs a real failing function to prove the asynchronous retries wait as they do on AWS (scaled down by@@ -41,7 +35,6 @@ class LambdaAsyncRetryDockerIntegrationTest {private static final String ACCOUNT = "000000000000";private static final String ROLE = "arn:aws:iam::" + ACCOUNT + ":role/lambda-role";private static final Duration RECEIVE_DEADLINE = Duration.ofSeconds(60);- private static final ObjectMapper MAPPER = new ObjectMapper();@InjectDockerClient dockerClient;@@ -68,7 +61,7 @@ class LambdaAsyncRetryDockerIntegrationTest {.then().statusCode(202);- JsonNode record = awaitRecord(fixture.aliasQueueUrl());+ JsonNode record = fixture.aliasQueue().awaitJson(RECEIVE_DEADLINE);long elapsedMillis = Duration.ofNanos(System.nanoTime() - started).toMillis();assertTrue(elapsedMillis >= 3000,@@ -79,7 +72,7 @@ class LambdaAsyncRetryDockerIntegrationTest {assertTrue(functionArn.endsWith(":1"), "functionArn should name the executed version: " + functionArn);assertEquals("1", record.path("responseContext").path("executedVersion").asText());assertEquals("alias", record.path("requestPayload").path("probe").asText());- assertTrue(receiveOnce(fixture.latestQueueUrl()).isEmpty(),+ assertTrue(fixture.latestQueue().receive().isEmpty(),"the unqualified OnFailure destination should receive nothing");} finally {deleteFunction(fixture.functionName());@@ -98,12 +91,12 @@ class LambdaAsyncRetryDockerIntegrationTest {.then().statusCode(202);- JsonNode record = awaitRecord(fixture.latestQueueUrl());+ JsonNode record = fixture.latestQueue().awaitJson(RECEIVE_DEADLINE);assertEquals("RetriesExhausted", record.path("requestContext").path("condition").asText());assertEquals(1, record.path("requestContext").path("approximateInvokeCount").asInt());assertEquals("latest", record.path("requestPayload").path("probe").asText());- assertTrue(receiveOnce(fixture.aliasQueueUrl()).isEmpty(),+ assertTrue(fixture.aliasQueue().receive().isEmpty(),"the alias OnFailure destination should receive nothing");} finally {deleteFunction(fixture.functionName());@@ -113,8 +106,8 @@ class LambdaAsyncRetryDockerIntegrationTest {private Fixture setUp() throws Exception {String suffix = UUID.randomUUID().toString().substring(0, 8);String functionName = "async-retry-" + suffix;- String latestQueueUrl = createQueue("async-retry-latest-" + suffix);- String aliasQueueUrl = createQueue("async-retry-alias-" + suffix);+ SqsTestQueue latestQueue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "async-retry-latest-");+ SqsTestQueue aliasQueue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "async-retry-alias-");given().contentType("application/json")@@ -152,7 +145,7 @@ class LambdaAsyncRetryDockerIntegrationTest {given().contentType("application/json").body("{\"MaximumRetryAttempts\":0,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"- .formatted(queueArn(latestQueueUrl)))+ .formatted(latestQueue.arn())).when().put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config").then()@@ -161,39 +154,13 @@ class LambdaAsyncRetryDockerIntegrationTest {given().contentType("application/json").body("{\"MaximumRetryAttempts\":2,\"DestinationConfig\":{\"OnFailure\":{\"Destination\":\"%s\"}}}"- .formatted(queueArn(aliasQueueUrl)))+ .formatted(aliasQueue.arn())).when().put(CONFIG_PATH + "/functions/" + functionName + "/event-invoke-config?Qualifier=live").then().statusCode(200);- return new Fixture(functionName, latestQueueUrl, aliasQueueUrl);- }-- private String createQueue(String queueName) {- Queue queue = RequestScopes.callAs(ACCOUNT, () -> sqsService.createQueue(queueName, Map.of(), REGION));- return queue.getQueueUrl();- }-- private String queueArn(String queueUrl) {- return RequestScopes.callAs(ACCOUNT, () ->- sqsService.getQueueAttributes(queueUrl, List.of("QueueArn"), REGION).get("QueueArn"));- }-- private List<Message> receiveOnce(String queueUrl) {- return RequestScopes.callAs(ACCOUNT, () -> sqsService.receiveMessage(queueUrl, 1, 30, 0, REGION));- }-- private JsonNode awaitRecord(String queueUrl) throws Exception {- long deadline = System.nanoTime() + RECEIVE_DEADLINE.toNanos();- while (System.nanoTime() < deadline) {- List<Message> messages = receiveOnce(queueUrl);- if (!messages.isEmpty()) {- return MAPPER.readTree(messages.get(0).getBody());- }- Thread.sleep(250);- }- return fail("no OnFailure record reached " + queueUrl + " within " + RECEIVE_DEADLINE.toSeconds() + " s");+ return new Fixture(functionName, latestQueue, aliasQueue);}private static void deleteFunction(String functionName) {@@ -221,6 +188,6 @@ class LambdaAsyncRetryDockerIntegrationTest {}}- private record Fixture(String functionName, String latestQueueUrl, String aliasQueueUrl) {+ private record Fixture(String functionName, SqsTestQueue latestQueue, SqsTestQueue aliasQueue) {}}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.javanew file mode 100644index 0000000000..2177368627--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresAccountIntegrationTest.java@@ -0,0 +1,83 @@+package io.github.hectorvent.floci.services.lambda;++import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.testing.SqsTestQueue;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.Map;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * A poller thread carries no request context, so account-aware stores fall back to the default+ * account. A mapping outside it must still reach its SQS and SNS destinations, as the poller sends+ * from such a thread. The test thread has no request context either.+ */+@QuarkusTest+class StreamBatchFailuresAccountIntegrationTest {++ private static final String ACCOUNT = "111122223333";+ private static final String REGION = "us-east-1";+ private static final Duration DELIVERY_TIMEOUT = Duration.ofSeconds(5);+ private static final StreamBatchFailures.DiscardedBatch BATCH = new StreamBatchFailures.DiscardedBatch(+ "DDBStreamBatchInfo", "shardId-000000000000", "s1", "s1",+ "2026-10-20T12:07:26Z", "2026-10-20T12:07:26Z", 1,+ "arn:aws:dynamodb:us-east-1:111122223333:table/t/stream/2026-08-01T00:00:00.000", () -> "{\"Records\":[]}");++ @Inject+ StreamBatchFailures failures;++ @Inject+ SqsService sqsService;++ @Inject+ SnsService snsService;++ @Test+ void sqsDestinationOutsideTheDefaultAccountGetsTheRecord() throws Exception {+ SqsTestQueue queue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "esm-failures-");++ failures.sendToOnFailureDestination(mappingTo(queue.arn()), BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ assertEquals("RetryAttemptsExhausted",+ queue.awaitJson(DELIVERY_TIMEOUT).at("/requestContext/condition").asText());+ }++ @Test+ void snsDestinationOutsideTheDefaultAccountGetsTheRecord() throws Exception {+ SqsTestQueue queue = SqsTestQueue.create(sqsService, ACCOUNT, REGION, "esm-failures-");+ String topicArn = RequestScopes.callAs(ACCOUNT, () -> snsService.createTopic(+ "esm-failures-" + UUID.randomUUID().toString().substring(0, 8), Map.of(), null, REGION).getTopicArn());+ RequestScopes.runAs(ACCOUNT, () -> snsService.subscribe(topicArn, "sqs", queue.arn(), REGION,+ Map.of("RawMessageDelivery", "true")));++ failures.sendToOnFailureDestination(mappingTo(topicArn), BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ assertEquals("RetryAttemptsExhausted",+ queue.awaitJson(DELIVERY_TIMEOUT).at("/requestContext/condition").asText());+ }++ private static EventSourceMapping mappingTo(String destinationArn) {+ EventSourceMapping esm = new EventSourceMapping();+ esm.setUuid("esm-1");+ esm.setAccountId(ACCOUNT);+ esm.setRegion(REGION);+ esm.setEventSourceArn(BATCH.streamArn());+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination(destinationArn);+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();+ destinationConfig.setOnFailure(onFailure);+ esm.setDestinationConfig(destinationConfig);+ return esm;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.javanew file mode 100644index 0000000000..f4db37e5bd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/StreamBatchFailuresTest.java@@ -0,0 +1,236 @@+package io.github.hectorvent.floci.services.lambda;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.sns.SnsService;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.testutil.LogCapture;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import org.mockito.ArgumentCaptor;++import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.function.Supplier;+import java.util.logging.Level;+import java.util.logging.LogRecord;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertSame;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.when;++class StreamBatchFailuresTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();+ private static final String STREAM_ARN = "arn:aws:dynamodb:us-east-1:000000000000:table/t/stream/2026-08-01T00:00:00.000";+ private static final String EVENT = "{\"Records\":[{\"eventSource\":\"aws:dynamodb\"}]}";+ /** Whole seconds: {@code ISO_INSTANT} would drop the fraction that AWS always writes. */+ private static final StreamBatchFailures.DiscardedBatch BATCH = batch(() -> EVENT);++ private SqsService sqsService;+ private SnsService snsService;+ private S3Service s3Service;+ private StreamBatchFailures failures;++ @BeforeEach+ void setUp() {+ EmulatorConfig config = mock(EmulatorConfig.class);+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ sqsService = mock(SqsService.class);+ snsService = mock(SnsService.class);+ s3Service = mock(S3Service.class);+ failures = new StreamBatchFailures(sqsService, snsService, s3Service, MAPPER, config);+ }++ @Test+ void s3KeyUsesTheUtcLayout() {+ UUID randomId = UUID.fromString("12345678-1234-1234-1234-123456789abc");++ String key = StreamBatchFailures.s3Key("esm-1", "shard-1", Instant.parse("2026-09-14T23:07:08Z"), randomId);++ assertEquals("aws/lambda/esm-1/shard-1/2026/09/14/"+ + "2026-09-14T23.07.08-12345678-1234-1234-1234-123456789abc", key);+ }++ @Test+ void retryBackoffDoublesFromThePollIntervalAndCapsAtOneMinute() {+ assertEquals(1_000, StreamBatchFailures.retryBackoffMs(500, 1));+ assertEquals(2_000, StreamBatchFailures.retryBackoffMs(500, 2));+ assertEquals(32_000, StreamBatchFailures.retryBackoffMs(500, 6));+ assertEquals(60_000, StreamBatchFailures.retryBackoffMs(500, 7));+ assertEquals(60_000, StreamBatchFailures.retryBackoffMs(500, 10_000), "a large retry count must not overflow");+ }++ @Test+ void retryAttemptsRunOutAfterOnePlusTheMaximumInvocations() {+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(null, null), 10_001), "unset means no limit");+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(-1, null), 10_001), "-1 means no limit");+ assertTrue(StreamBatchFailures.retryAttemptsExhausted(esm(0, null), 1));+ assertFalse(StreamBatchFailures.retryAttemptsExhausted(esm(10, null), 10));+ assertTrue(StreamBatchFailures.retryAttemptsExhausted(esm(10, null), 11));+ }++ @Test+ void maximumRecordAgeIsExceededOnlyForAFiniteLimit() {+ Instant arrival = Instant.parse("2026-10-20T12:07:26.012Z");+ long weekLater = arrival.plusSeconds(604_801).toEpochMilli();+ long almostAMinute = arrival.toEpochMilli() + 59_999;+ long aMinute = arrival.toEpochMilli() + 60_000;++ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, null), arrival, weekLater), "unset");+ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, -1), arrival, weekLater), "-1");+ assertFalse(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, 60), arrival, almostAMinute));+ assertTrue(StreamBatchFailures.maximumRecordAgeExceeded(esm(null, 60), arrival, aMinute));+ }++ @Test+ void sqsRecordFillsDefaultsWritesTheTimestampInMillisecondsAndDoesNotBuildTheEvent() throws Exception {+ EventSourceMapping esm = esm(0, null);+ esm.setDestinationConfig(destination("arn:aws:sqs:us-east-1:000000000000:dlq"));+ StreamBatchFailures.DiscardedBatch batch = batch(() -> {+ throw new AssertionError("only an S3 destination may build the event");+ });++ failures.sendToOnFailureDestination(esm, batch, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ ArgumentCaptor<String> body = ArgumentCaptor.forClass(String.class);+ verify(sqsService, times(1)).sendMessage(eq("http://localhost:4566/000000000000/dlq"), body.capture(),+ eq(0), eq("us-east-1"));+ JsonNode record = MAPPER.readTree(body.getValue());+ assertEquals("", record.at("/requestContext/requestId").asText(), record.toString());+ assertEquals("", record.at("/requestContext/functionArn").asText());+ assertEquals(List.of("statusCode", "executedVersion"), fieldNames(record.path("responseContext")),+ "no functionError without a function error: " + record);+ assertEquals(200, record.at("/responseContext/statusCode").asInt());+ assertEquals("$LATEST", record.at("/responseContext/executedVersion").asText());+ assertEquals("2023-11-14T22:13:20Z", record.at("/DDBStreamBatchInfo/approximateArrivalOfFirstRecord").asText(),+ "the arrival times are written as the source gives them");+ assertEquals("2023-11-14T22:13:21Z", record.at("/DDBStreamBatchInfo/approximateArrivalOfLastRecord").asText());+ assertTrue(record.path("timestamp").asText().matches("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z"),+ "timestamp must be ISO-8601 with exactly three fraction digits: " + record.path("timestamp"));+ }++ @Test+ void destinationInAnotherRegionIsReachedInItsOwnRegion() {+ EventSourceMapping esm = esm(0, null);+ esm.setDestinationConfig(destination("arn:aws:sqs:eu-west-2:000000000000:dlq"));++ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ verify(sqsService, times(1)).sendMessage(eq("http://localhost:4566/000000000000/dlq"), anyString(),+ eq(0), eq("eu-west-2"));+ }++ @Test+ void s3RecordCarriesTheEventAsAPayloadString() throws Exception {+ EventSourceMapping esm = esm(0, null);+ esm.setDestinationConfig(destination("arn:aws:s3:::failures"));+ InvokeResult error = new InvokeResult(200, "Unhandled", new byte[0], null, "req-1");++ failures.sendToOnFailureDestination(esm, BATCH, error, 1, StreamBatchFailures.MAXIMUM_RECORD_AGE_EXCEEDED);++ ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);+ ArgumentCaptor<byte[]> body = ArgumentCaptor.forClass(byte[].class);+ verify(s3Service, times(1)).putObject(eq("failures"), key.capture(), body.capture(),+ eq("application/json"), eq(Map.of()));+ assertTrue(key.getValue().startsWith("aws/lambda/esm-1/shardId-000000000000/"), key.getValue());+ JsonNode record = MAPPER.readTree(new String(body.getValue(), StandardCharsets.UTF_8));+ assertEquals(List.of("requestContext", "responseContext", "version", "payload", "timestamp", "DDBStreamBatchInfo"),+ fieldNames(record));+ assertEquals(EVENT, record.path("payload").asText());+ assertEquals("MaximumRecordAgeExceeded", record.at("/requestContext/condition").asText());+ assertEquals("Unhandled", record.at("/responseContext/functionError").asText());+ verifyNoInteractions(sqsService, snsService);+ }++ @Test+ void refusedSendIsLoggedWithItsCauseNotThrown() {+ EventSourceMapping esm = esm(0, null);+ esm.setDestinationConfig(destination("arn:aws:sqs:us-east-1:000000000000:gone"));+ AwsException refusal = new AwsException("AWS.SimpleQueueService.NonExistentQueue",+ "The specified queue does not exist.", 400);+ when(sqsService.sendMessage(anyString(), anyString(), anyInt(), anyString())).thenThrow(refusal);++ List<LogRecord> records = LogCapture.capture(StreamBatchFailures.class, () ->+ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED));++ LogRecord error = records.stream()+ .filter(record -> record.getLevel().intValue() >= Level.SEVERE.intValue())+ .findFirst()+ .orElseThrow(() -> new AssertionError("no error was logged: " + records));+ assertSame(refusal, error.getThrown(), "the error log must keep the cause and its stack trace");+ }++ @ParameterizedTest+ @ValueSource(strings = {"", "arn:aws:lambda:us-east-1:000000000000:function:other", "arn:aws:s3:::"})+ void unusableDestinationSendsNothing(String destinationArn) {+ EventSourceMapping esm = esm(0, null);+ esm.setDestinationConfig(destination(destinationArn));++ failures.sendToOnFailureDestination(esm, BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ verifyNoInteractions(sqsService, snsService, s3Service);+ }++ @Test+ void mappingWithoutDestinationSendsNothing() {+ failures.sendToOnFailureDestination(esm(0, null), BATCH, new InvokeResult(), 1,+ StreamBatchFailures.RETRY_ATTEMPTS_EXHAUSTED);++ verifyNoInteractions(sqsService, snsService, s3Service);+ }++ private static StreamBatchFailures.DiscardedBatch batch(Supplier<String> event) {+ return new StreamBatchFailures.DiscardedBatch("DDBStreamBatchInfo", "shardId-000000000000", "s1", "s2",+ "2023-11-14T22:13:20Z", "2023-11-14T22:13:21Z", 2, STREAM_ARN, event);+ }++ private static EventSourceMapping esm(Integer maximumRetryAttempts, Integer maximumRecordAgeInSeconds) {+ EventSourceMapping esm = new EventSourceMapping();+ esm.setUuid("esm-1");+ esm.setAccountId("000000000000");+ esm.setRegion("us-east-1");+ esm.setEventSourceArn(STREAM_ARN);+ esm.setMaximumRetryAttempts(maximumRetryAttempts);+ esm.setMaximumRecordAgeInSeconds(maximumRecordAgeInSeconds);+ return esm;+ }++ private static EventSourceMapping.DestinationConfig destination(String arn) {+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination(arn);+ EventSourceMapping.DestinationConfig destinationConfig = new EventSourceMapping.DestinationConfig();+ destinationConfig.setOnFailure(onFailure);+ return destinationConfig;+ }++ private static List<String> fieldNames(JsonNode node) {+ List<String> names = new ArrayList<>();+ node.fieldNames().forEachRemaining(names::add);+ return names;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.java b/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.javanew file mode 100644index 0000000000..9a7497d151--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/testing/SqsTestQueue.java@@ -0,0 +1,74 @@+package io.github.hectorvent.floci.testing;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.sqs.SqsService;+import io.github.hectorvent.floci.services.sqs.model.Message;++import java.time.Duration;+import java.util.List;+import java.util.Map;+import java.util.UUID;++import static org.awaitility.Awaitility.await;++/**+ * An SQS queue a test creates in one account and Region, typically as a destination, and then waits+ * on for one JSON message. Every call runs in that account, so it works from a thread without a+ * request context.+ */+public final class SqsTestQueue {++ /** Longer than any test, so a received message never comes back and fails a later check. */+ private static final int VISIBILITY_TIMEOUT_SECONDS = 300;+ private static final Duration POLL_INTERVAL = Duration.ofMillis(250);+ private static final ObjectMapper MAPPER = new ObjectMapper();++ private final SqsService sqsService;+ private final String account;+ private final String region;+ private final String url;++ private SqsTestQueue(SqsService sqsService, String account, String region, String url) {+ this.sqsService = sqsService;+ this.account = account;+ this.region = region;+ this.url = url;+ }++ /** Creates a queue named {@code namePrefix} plus a random suffix. */+ public static SqsTestQueue create(SqsService sqsService, String account, String region, String namePrefix) {+ String name = namePrefix + UUID.randomUUID().toString().substring(0, 8);+ String url = RequestScopes.callAs(account, () -> sqsService.createQueue(name, Map.of(), region).getQueueUrl());+ return new SqsTestQueue(sqsService, account, region, url);+ }++ public String url() {+ return url;+ }++ public String arn() {+ return RequestScopes.callAs(account, () ->+ sqsService.getQueueAttributes(url, List.of("QueueArn"), region).get("QueueArn"));+ }++ /** At most one message, received without waiting. */+ public List<Message> receive() {+ return RequestScopes.callAs(account, () ->+ sqsService.receiveMessage(url, 1, VISIBILITY_TIMEOUT_SECONDS, 0, region));+ }++ /** Polls until a message arrives, and returns its body parsed as JSON. */+ public JsonNode awaitJson(Duration timeout) throws Exception {+ List<Message> messages = await("a message in " + url)+ .atMost(timeout)+ .pollInterval(POLL_INTERVAL)+ .until(this::receive, received -> !received.isEmpty());+ return MAPPER.readTree(messages.get(0).getBody());+ }++ public void delete() {+ RequestScopes.runAs(account, () -> sqsService.deleteQueue(url, region));+ }+}