Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): deliver DynamoDB stream failures to S3 (#3622)
XuJian561aa32283e4
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 168dd30966..d5d1177da4 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;@@ -15,6 +16,7 @@ import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.pipes.PipesFilterMatcher;+import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.sns.SnsService;import io.github.hectorvent.floci.services.sqs.SqsService;import io.vertx.core.Vertx;@@ -23,7 +25,9 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.nio.charset.StandardCharsets;import java.time.Instant;+import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;import java.util.ArrayList;import java.util.HashMap;@@ -31,6 +35,7 @@ import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;@@ -39,6 +44,10 @@ import java.util.concurrent.Executors;public class DynamoDbStreamsEventSourcePoller implements Resettable {private static final Logger LOG = Logger.getLogger(DynamoDbStreamsEventSourcePoller.class);+ private static final DateTimeFormatter S3_ON_FAILURE_PATH_DATE_FORMATTER =+ DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);+ private static final DateTimeFormatter S3_ON_FAILURE_FILE_TIME_FORMATTER =+ DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH.mm.ss").withZone(ZoneOffset.UTC);/** Raised by the stream when a stored checkpoint has aged out of the retained window. */private static final String TRIMMED_DATA_ACCESS_EXCEPTION = "TrimmedDataAccessException";@@ -52,6 +61,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {private final PipesFilterMatcher filterMatcher;private final SqsService sqsService;private final SnsService snsService;+ private final S3Service s3Service;private final String baseUrl;private final long pollIntervalMs;private final ConcurrentHashMap<String, Long> timerIds = new ConcurrentHashMap<>();@@ -72,7 +82,8 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {EmulatorConfig config,PipesFilterMatcher filterMatcher,SqsService sqsService,- SnsService snsService) {+ SnsService snsService,+ S3Service s3Service) {this.vertx = vertx;this.streamService = streamService;this.executorService = executorService;@@ -84,6 +95,7 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {this.filterMatcher = filterMatcher;this.sqsService = sqsService;this.snsService = snsService;+ this.s3Service = s3Service;}public void startPersistedPollers() {@@ -400,16 +412,30 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}try {- String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);- String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());-if (destinationArn.contains(":sqs:")) {+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());String queueUrl = AwsArnUtils.arnToQueueUrl(destinationArn, baseUrl);+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);sqsService.sendMessage(queueUrl, payload, 0, region);LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SQS DLQ {1}", esm.getUuid(), destinationArn);} else if (destinationArn.contains(":sns:")) {+ String region = AwsArnUtils.regionOrDefault(destinationArn, esm.getRegion());+ String payload = buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount);snsService.publish(destinationArn, null, payload, "ESM OnFailure", region);LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to SNS DLQ {1}", esm.getUuid(), destinationArn);+ } else if (destinationArn.contains(":s3:")) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(destinationArn);+ if (!"s3".equals(arn.service()) || arn.resource().isBlank()) {+ throw new IllegalArgumentException("Invalid S3 destination ARN: " + destinationArn);+ }+ String key = buildS3OnFailureKey(esm.getUuid(), shardId, Instant.now(), UUID.randomUUID());+ String s3Payload = buildS3OnFailurePayload(esm, shardId, records, invokeResult, invokeCount);+ RequestScopes.runAs(esm.getAccountId(), () ->+ s3Service.putObject(arn.resource(), key,+ s3Payload.getBytes(StandardCharsets.UTF_8),+ "application/json", Map.of()));+ LOG.infov("DynamoDB Streams ESM {0}: sent failed batch to S3 bucket {1} with key {2}",+ esm.getUuid(), arn.resource(), key);} else {LOG.warnv("DynamoDB Streams ESM {0}: unsupported OnFailure destination ARN {1}",esm.getUuid(), destinationArn);@@ -420,6 +446,26 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {}}+ private String buildS3OnFailurePayload(EventSourceMapping esm, String shardId,+ List<DynamoDbStreamRecord> records,+ InvokeResult invokeResult,+ int invokeCount) {+ try {+ ObjectNode root = (ObjectNode) objectMapper.readTree(+ buildOnFailurePayload(esm, shardId, records, invokeResult, invokeCount));+ root.put("payload", buildDynamoDbEvent(records, esm));+ return objectMapper.writeValueAsString(root);+ } catch (Exception e) {+ throw new IllegalStateException("Failed to serialize S3 OnFailure payload", e);+ }+ }++ static String buildS3OnFailureKey(String esmUuid, String shardId, Instant now, UUID randomId) {+ return "aws/lambda/" + esmUuid + "/" + shardId + "/"+ + S3_ON_FAILURE_PATH_DATE_FORMATTER.format(now) + "/"+ + S3_ON_FAILURE_FILE_TIME_FORMATTER.format(now) + "-" + randomId;+ }+private String buildOnFailurePayload(EventSourceMapping esm, String shardId,List<DynamoDbStreamRecord> records,InvokeResult invokeResult,diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex f21f021a5b..dad0016048 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -5,24 +5,43 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;+import io.github.hectorvent.floci.services.eventbridge.EventBridgeService;+import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.pipes.PipesFilterMatcher;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.Bucket;+import io.github.hectorvent.floci.services.s3.model.ObjectAnnotation;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.github.hectorvent.floci.services.s3.model.S3ObjectUpdatedEvent;+import io.quarkus.test.junit.QuarkusTest;import io.vertx.core.Vertx;+import jakarta.enterprise.event.Event;+import jakarta.enterprise.inject.Instance;+import jakarta.inject.Inject;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;import java.time.Instant;import java.util.ArrayList;import java.util.List;+import java.util.Map;import java.util.Optional;+import java.util.UUID;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -46,10 +65,13 @@ import static org.mockito.Mockito.when;* (in-memory only), so after a restart the sequence numbers restart from 1 and a stale checkpoint* would silently skip every new record.*/+@QuarkusTestclass DynamoDbStreamsEventSourcePollerTest {private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();private static final String ACCOUNT_ID = "000000000000";+ private static final String AMBIENT_ACCOUNT_ID = "222233334444";+ private static final String BUCKET_OWNER_ACCOUNT_ID = "111122223333";private static final String STREAM_ARN ="arn:aws:dynamodb:us-east-1:000000000000:table/t/stream/2026-08-01T00:00:00.000";private static final String STALE_CHECKPOINT = "000000000000000000634";@@ -63,6 +85,10 @@ class DynamoDbStreamsEventSourcePollerTest {private PipesFilterMatcher filterMatcher;private io.github.hectorvent.floci.services.sqs.SqsService sqsService;private io.github.hectorvent.floci.services.sns.SnsService snsService;+ private io.github.hectorvent.floci.services.s3.S3Service s3Service;++ @Inject+ Instance<RequestContext> requestContextInstance;@BeforeEachvoid setUp() {@@ -81,12 +107,13 @@ class DynamoDbStreamsEventSourcePollerTest {filterMatcher = new PipesFilterMatcher(OBJECT_MAPPER);sqsService = mock(io.github.hectorvent.floci.services.sqs.SqsService.class);snsService = mock(io.github.hectorvent.floci.services.sns.SnsService.class);+ s3Service = mock(io.github.hectorvent.floci.services.s3.S3Service.class);// A mocked Vertx makes setPeriodic a no-op, so startPolling registers no live timer and// the tests drive pollAndInvoke deterministically.poller = new DynamoDbStreamsEventSourcePoller(mock(Vertx.class), streamService, executorService, functionStore,- esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService);+ esmStore, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);}private EventSourceMapping persistedStreamsEsmWithStaleCheckpoint() {@@ -160,7 +187,7 @@ class DynamoDbStreamsEventSourcePollerTest {private DynamoDbStreamsEventSourcePoller pollerWith(EsmStore store) {return new DynamoDbStreamsEventSourcePoller(mock(Vertx.class), streamService, executorService, functionStore,- store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService);+ store, OBJECT_MAPPER, config, filterMatcher, sqsService, snsService, s3Service);}/**@@ -631,6 +658,141 @@ class DynamoDbStreamsEventSourcePollerTest {assertFalse(dlqPayload.has("hasBeenTruncated"));}+ @Test+ void maxRetryAttemptsExhaustedUsesEsmAccountOutsideRequestContext() throws Exception {+ S3OnFailureDelivery delivery = deliverFailedBatchToS3(ACCOUNT_ID, false);++ assertS3FailureRecord(delivery, ACCOUNT_ID);+ assertTrue(delivery.objects().keysForAccount(AMBIENT_ACCOUNT_ID).isEmpty(),+ "the async/default account must not receive a shadow failure object");+ }++ @Test+ void maxRetryAttemptsExhaustedDeliversOriginalEventToCrossAccountBucketOwner() throws Exception {+ S3OnFailureDelivery delivery = deliverFailedBatchToS3(BUCKET_OWNER_ACCOUNT_ID, true);++ assertS3FailureRecord(delivery, BUCKET_OWNER_ACCOUNT_ID);+ assertTrue(delivery.objects().keysForAccount(ACCOUNT_ID).isEmpty(),+ "the ESM account must not receive a shadow object for a cross-account bucket");+ assertTrue(delivery.objects().keysForAccount(AMBIENT_ACCOUNT_ID).isEmpty(),+ "the async/default account must not receive a shadow failure object");+ }++ private S3OnFailureDelivery deliverFailedBatchToS3(+ String bucketOwnerAccountId, boolean globalBucketNamespace) {+ stubTrimHorizon(List.of(ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}")));+ InvokeResult err = new InvokeResult();+ err.setFunctionError("Unhandled");+ err.setStatusCode(200);+ err.setRequestId("req-s3");+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(err);++ EsmStore store = mock(EsmStore.class);+ EventSourceMapping esm = filterEsm();+ esm.setMaximumRetryAttempts(0);++ EventSourceMapping.DestinationConfig destConfig = new EventSourceMapping.DestinationConfig();+ EventSourceMapping.OnFailure onFailure = new EventSourceMapping.OnFailure();+ onFailure.setDestination("arn:aws:s3:::my-failed-events");+ destConfig.setOnFailure(onFailure);+ esm.setDestinationConfig(destConfig);++ AccountAwareStorageBackend<Bucket> buckets = new AccountAwareStorageBackend<>(+ new InMemoryStorage<>(), requestContextInstance, AMBIENT_ACCOUNT_ID);+ AccountAwareStorageBackend<S3Object> objects = new AccountAwareStorageBackend<>(+ new InMemoryStorage<>(), requestContextInstance, AMBIENT_ACCOUNT_ID);+ AccountAwareStorageBackend<ObjectAnnotation> annotations = new AccountAwareStorageBackend<>(+ new InMemoryStorage<>(), requestContextInstance, AMBIENT_ACCOUNT_ID);+ AccountAwareStorageBackend<String> accountPublicAccessBlocks = new AccountAwareStorageBackend<>(+ new InMemoryStorage<>(), requestContextInstance, AMBIENT_ACCOUNT_ID);+ buckets.putForAccount(bucketOwnerAccountId, "my-failed-events", new Bucket("my-failed-events"));++ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.create(eq("s3"), anyString(), any())).thenAnswer(invocation -> {+ String fileName = invocation.getArgument(1);+ return switch (fileName) {+ case "s3-buckets.json" -> buckets;+ case "s3-objects.json" -> objects;+ case "s3-annotations.json" -> annotations;+ case "s3-account-public-access-block.json" -> accountPublicAccessBlocks;+ default -> throw new AssertionError("Unexpected S3 store: " + fileName);+ };+ });++ EmulatorConfig.StorageConfig storageConfig = mock(EmulatorConfig.StorageConfig.class);+ when(storageConfig.persistentPath()).thenReturn(Path.of("target", "s3-onfailure-test").toString());+ EmulatorConfig.ServiceStorageOverrides storageOverrides = mock(EmulatorConfig.ServiceStorageOverrides.class);+ EmulatorConfig.S3StorageConfig s3StorageConfig = mock(EmulatorConfig.S3StorageConfig.class);+ when(storageConfig.services()).thenReturn(storageOverrides);+ when(storageOverrides.s3()).thenReturn(s3StorageConfig);+ when(s3StorageConfig.mode()).thenReturn(Optional.of("memory"));+ when(config.storage()).thenReturn(storageConfig);+ EmulatorConfig.S3ServiceConfig s3Config = mock(EmulatorConfig.S3ServiceConfig.class);+ when(s3Config.globalBucketNamespace()).thenReturn(globalBucketNamespace);+ when(config.services().s3()).thenReturn(s3Config);++ RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getAccountId()).thenAnswer(+ invocation -> requestContextInstance.get().getAccountId());+ s3Service = new S3Service(+ storageFactory, config,+ mock(io.github.hectorvent.floci.services.sqs.SqsService.class),+ mock(io.github.hectorvent.floci.services.sns.SnsService.class),+ mock(Instance.class), mock(EventBridgeService.class), mock(Event.class),+ regionResolver, OBJECT_MAPPER, mock(IamService.class));++ pollerWith(store).pollAndInvoke(esm);++ verify(store, timeout(2000)).saveForAccount(eq(ACCOUNT_ID), any());+ assertEquals("s1", esm.getShardSequenceNumbers().get(DynamoDbStreamService.SHARD_ID));++ return new S3OnFailureDelivery(s3Service, objects);+ }++ private void assertS3FailureRecord(S3OnFailureDelivery delivery, String expectedOwnerAccountId)+ throws Exception {+ String storedObjectKey = delivery.objects().keysForAccount(expectedOwnerAccountId).stream()+ .filter(key -> key.startsWith("my-failed-events/aws/lambda/esm-f/"))+ .findFirst()+ .orElseThrow(() -> new AssertionError("failure object was not stored in the bucket owner's account"));++ String key = storedObjectKey.substring("my-failed-events/".length());+ byte[] body = RequestScopes.callAs(ACCOUNT_ID,+ () -> delivery.s3Service().getObject("my-failed-events", key).getData());++ assertTrue(key.matches(+ "^aws/lambda/esm-f/shardId-0000000001-00000000001/\\d{4}/\\d{2}/\\d{2}/"+ + "\\d{4}-\\d{2}-\\d{2}T\\d{2}\\.\\d{2}\\.\\d{2}-[0-9a-f-]{36}$"));++ JsonNode failureRecord = OBJECT_MAPPER.readTree(new String(body, StandardCharsets.UTF_8));+ assertEquals("RetryAttemptsExhausted",+ failureRecord.path("requestContext").path("condition").asText());+ assertEquals(1, failureRecord.path("requestContext").path("approximateInvokeCount").asInt());+ assertEquals("req-s3", failureRecord.path("requestContext").path("requestId").asText());+ assertTrue(failureRecord.path("payload").isTextual());++ JsonNode originalEvent = OBJECT_MAPPER.readTree(failureRecord.path("payload").asText());+ assertEquals(1, originalEvent.path("Records").size());+ assertEquals("s1", originalEvent.path("Records").get(0)+ .path("dynamodb").path("SequenceNumber").asText());+ }++ private record S3OnFailureDelivery(+ S3Service s3Service, AccountAwareStorageBackend<S3Object> objects) {+ }++ @Test+ void s3OnFailureKeyUsesUtcLayout() {+ UUID randomId = UUID.fromString("12345678-1234-1234-1234-123456789abc");++ String key = DynamoDbStreamsEventSourcePoller.buildS3OnFailureKey(+ "esm-1", "shard-1", Instant.parse("2026-09-14T23:07:08Z"), randomId);++ assertEquals("aws/lambda/esm-1/shard-1/2026/09/14/"+ + "2026-09-14T23.07.08-12345678-1234-1234-1234-123456789abc", key);+ }+@Testvoid onFailurePayloadUsesEpochSecondsNotMillisForArrivalTimestamps() throws Exception {// Chosen so the seconds/millis interpretations land 46+ years apart: a millis bug cannot