Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): mark time-to-live deletions on the stream as AWS does (#4930)
Mario Toffia9a7ec1c9d0b8
diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex 4a6cb9ec37..a19956d461 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -49,6 +49,7 @@ import software.amazon.awssdk.services.datasync.DataSyncClient;import software.amazon.awssdk.services.detective.DetectiveClient;import software.amazon.awssdk.services.dlm.DlmClient;import software.amazon.awssdk.services.dynamodb.DynamoDbClient;+import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClient;import software.amazon.awssdk.services.ec2.Ec2Client;import software.amazon.awssdk.services.ecr.EcrClient;import software.amazon.awssdk.services.ecs.EcsClient;@@ -713,6 +714,14 @@ public final class TestFixtures {.build();}+ public static DynamoDbStreamsClient dynamoDbStreamsClient() {+ return DynamoDbStreamsClient.builder()+ .endpointOverride(ENDPOINT)+ .region(REGION)+ .credentialsProvider(CREDENTIALS)+ .build();+ }+public static LambdaClient lambdaClient() {return LambdaClient.builder().endpointOverride(ENDPOINT)diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.javanew file mode 100644index 0000000000..7f96a30e23--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.java@@ -0,0 +1,110 @@+package com.floci.test;++import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.DisplayName;+import org.junit.jupiter.api.Test;+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;+import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;+import software.amazon.awssdk.services.dynamodb.model.BillingMode;+import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;+import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;+import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest;+import software.amazon.awssdk.services.dynamodb.model.DescribeStreamRequest;+import software.amazon.awssdk.services.dynamodb.model.GetRecordsRequest;+import software.amazon.awssdk.services.dynamodb.model.GetShardIteratorRequest;+import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;+import software.amazon.awssdk.services.dynamodb.model.KeyType;+import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;+import software.amazon.awssdk.services.dynamodb.model.Record;+import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException;+import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;+import software.amazon.awssdk.services.dynamodb.model.Shard;+import software.amazon.awssdk.services.dynamodb.model.ShardIteratorType;+import software.amazon.awssdk.services.dynamodb.model.StreamSpecification;+import software.amazon.awssdk.services.dynamodb.model.StreamViewType;+import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClient;++import java.util.Map;+import java.util.Optional;++import static org.assertj.core.api.Assertions.assertThat;++/**+ * Reads stream records through the SDK's DynamoDB Streams client, whose JSON unmarshaller matches the+ * Streams API's capitalised members exactly.+ */+@DisplayName("DynamoDB Streams")+class DynamoDbStreamsTest {++ private static final String TABLE_NAME = TestFixtures.uniqueName("sdk-stream");++ private static DynamoDbClient ddb;+ private static DynamoDbStreamsClient streams;+ private static String streamArn;++ @BeforeAll+ static void setup() {+ ddb = TestFixtures.dynamoDbClient();+ streams = TestFixtures.dynamoDbStreamsClient();+ streamArn = ddb.createTable(CreateTableRequest.builder()+ .tableName(TABLE_NAME)+ .keySchema(KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build())+ .attributeDefinitions(AttributeDefinition.builder()+ .attributeName("pk").attributeType(ScalarAttributeType.S).build())+ .billingMode(BillingMode.PAY_PER_REQUEST)+ .streamSpecification(StreamSpecification.builder()+ .streamEnabled(true).streamViewType(StreamViewType.NEW_AND_OLD_IMAGES).build())+ .build())+ .tableDescription().latestStreamArn();+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)+ .item(Map.of("pk", AttributeValue.fromS("deleted")))+ .build());+ ddb.deleteItem(DeleteItemRequest.builder().tableName(TABLE_NAME)+ .key(Map.of("pk", AttributeValue.fromS("deleted")))+ .build());+ }++ @AfterAll+ static void cleanup() {+ if (ddb != null) {+ try {+ ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());+ } catch (ResourceNotFoundException ignored) {+ // The table is gone already when setup failed before creating it.+ }+ ddb.close();+ }+ if (streams != null) {+ streams.close();+ }+ }++ @Test+ @DisplayName("GetRecords returns a DeleteItem removal with no identity")+ void deleteItemRemovalCarriesNoIdentity() {+ Optional<Record> removal = findRemovalOf("deleted");++ assertThat(removal).hasValueSatisfying(record -> assertThat(record.userIdentity()).isNull());+ }++ private static Optional<Record> findRemovalOf(String pk) {+ for (Shard shard : streams.describeStream(DescribeStreamRequest.builder().streamArn(streamArn).build())+ .streamDescription().shards()) {+ String iterator = streams.getShardIterator(GetShardIteratorRequest.builder()+ .streamArn(streamArn)+ .shardId(shard.shardId())+ .shardIteratorType(ShardIteratorType.TRIM_HORIZON)+ .build()).shardIterator();+ for (Record record : streams.getRecords(GetRecordsRequest.builder().shardIterator(iterator).build())+ .records()) {+ if ("REMOVE".equals(record.eventNameAsString())+ && pk.equals(record.dynamodb().keys().get("pk").s())) {+ return Optional.of(record);+ }+ }+ }+ return Optional.empty();+ }+}diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 5b25fec9b8..c68030c9a6 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -70,6 +70,26 @@ epoch time in the past. Each deletion writes a `REMOVE` record to the table's stforwarded to an active Kinesis streaming destination. Between sweeps, reads already leave expireditems out.+As in AWS, the records of a TTL deletion carry the DynamoDB service identity, which a `DeleteItem`+removal does not. The member names depend on where the record is read. The DynamoDB Streams API+(`GetRecords`) capitalises them, as the SDK's `Identity` shape expects:++```json+"userIdentity": {"PrincipalId": "dynamodb.amazonaws.com", "Type": "Service"}+```++The records that Lambda event source mappings and EventBridge Pipes deliver and filter use the+lowercase names the DynamoDB developer guide shows, and so does the Kinesis streaming destination+record:++```json+"userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"}+```++A Lambda event source mapping or an EventBridge pipe that should see only TTL deletions filters on+it with the pattern+`{"userIdentity": {"type": ["Service"], "principalId": ["dynamodb.amazonaws.com"]}}`.+## DynamoDB Local backendBy default Floci stores tables in its own engine, the `native` backend. With@@ -128,6 +148,11 @@ region is rejected.`memory` storage they are lost when Floci restarts while the tables stay in DynamoDB Local.- **Stream consumers.** Lambda event source mappings, EventBridge Pipes and other Floci streamconsumers read the DynamoDB Local streams through the Streams API.+- **Time to live deletions.** DynamoDB Local deletes expired items itself and writes each deletion+ to the stream with the event name `UNKNOWN_TO_SDK_VERSION` and no `userIdentity`. Floci's+ `GetRecords` returns it as AWS writes it, a `REMOVE` record with the DynamoDB service+ `userIdentity` (see [Time to Live](#time-to-live)), so every stream consumer and event filter+ sees the AWS shape.### Limitsdiff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex c95b93aca0..879511d66c 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -2778,12 +2778,13 @@ public class DynamoDbService {}deletedForTable++;if (streamService != null) {- streamService.captureEvent("REMOVE", removed, null, scan.table(), scan.region());+ streamService.captureEvent("REMOVE", removed, null, scan.table(), scan.region(), true);}if (kinesisForwarder != null) {// Out of request scope here: pass the table owner's account explicitly so the CDC// record lands in the owner's stream, not the default account's same-named stream.- kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId());+ kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId(),+ true);}}if (deletedForTable > 0) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.javaindex 023737132e..5970c67750 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java@@ -157,6 +157,12 @@ public class DynamoDbStreamService {public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,TableDefinition table, String region) {+ captureEvent(eventName, oldItem, newItem, table, region, false);+ }++ /** {@code ttlDeletion} marks a removal made by the time to live sweep rather than by a caller. */+ public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,+ TableDefinition table, String region, boolean ttlDeletion) {StreamDescription sd = streams.get(table.getTableArn());if (sd == null || !"ENABLED".equals(sd.getStreamStatus())) {return;@@ -184,6 +190,7 @@ public class DynamoDbStreamService {record.setNewImage(newImage);record.setOldImage(oldImage);record.setStreamViewType(viewType);+ record.setTtlDeletion(ttlDeletion);ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(sd.getStreamArn());if (deque != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.javanew file mode 100644index 0000000000..d8b84ca6d4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.java@@ -0,0 +1,54 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.ServicePrincipals;++/**+ * The {@code userIdentity} AWS writes on the records of an item that time to live deleted. A consumer+ * tells a TTL deletion from a {@code DeleteItem} by it. Each output names the identity's members its+ * own way, and {@link Shape} holds the names of every output.+ */+public final class DynamoDbTtlIdentity {++ private static final String USER_IDENTITY = "userIdentity";++ /** An output a record is read from, with the member names its identity has there. */+ public enum Shape {+ /** The DynamoDB Streams API, whose {@code Identity} members AWS returns capitalised. */+ STREAMS_API("Type", "PrincipalId"),+ /**+ * The Lambda and EventBridge Pipes event record, and the Kinesis Data Streams destination+ * payload: lowercase names, as measured against AWS.+ */+ EVENT_RECORD("type", "principalId");++ private final String type;+ private final String principalId;++ Shape(String type, String principalId) {+ this.type = type;+ this.principalId = principalId;+ }+ }++ private DynamoDbTtlIdentity() {+ }++ public static void putOn(ObjectNode record, Shape shape) {+ record.putObject(USER_IDENTITY)+ .put(shape.type, "Service")+ .put(shape.principalId, ServicePrincipals.of("dynamodb"));+ }++ /** A copy of a Streams API record, its identity renamed to the event record shape Lambda and Pipes deliver. */+ public static ObjectNode toEventRecord(JsonNode streamsRecord) {+ ObjectNode record = streamsRecord.deepCopy();+ if (record.get(USER_IDENTITY) instanceof ObjectNode identity) {+ ObjectNode event = record.putObject(USER_IDENTITY);+ event.set(Shape.EVENT_RECORD.type, identity.get(Shape.STREAMS_API.type));+ event.set(Shape.EVENT_RECORD.principalId, identity.get(Shape.STREAMS_API.principalId));+ }+ return record;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.javaindex 8632558ddb..506ce16dad 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java@@ -140,6 +140,12 @@ public class KinesisStreamingForwarder {public void forward(String eventName, JsonNode oldItem, JsonNode newItem,TableDefinition table, String region, String ownerAccountId) {+ forward(eventName, oldItem, newItem, table, region, ownerAccountId, false);+ }++ /** {@code ttlDeletion} marks a removal made by the time to live sweep rather than by a caller. */+ public void forward(String eventName, JsonNode oldItem, JsonNode newItem,+ TableDefinition table, String region, String ownerAccountId, boolean ttlDeletion) {List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();if (destinations == null || destinations.isEmpty()) {return;@@ -169,7 +175,7 @@ public class KinesisStreamingForwarder {for (String precision : precisionByStreamArn.values()) {if (!dataByPrecision.containsKey(precision)) {ObjectNode payload = buildPayload(eventId, eventName, keys, newItem, oldItem,- table.getTableName(), region, now, precision);+ table.getTableName(), region, now, precision, ttlDeletion);dataByPrecision.put(precision, objectMapper.writeValueAsBytes(payload));}}@@ -441,12 +447,17 @@ public class KinesisStreamingForwarder {private ObjectNode buildPayload(String eventId, String eventName, JsonNode keys,JsonNode newImage, JsonNode oldImage,- String tableName, String region, Instant timestamp, String precision) {+ String tableName, String region, Instant timestamp, String precision,+ boolean ttlDeletion) {ObjectNode payload = objectMapper.createObjectNode();payload.put("awsRegion", region);payload.put("eventID", eventId);payload.put("eventName", eventName);- payload.putNull("userIdentity");+ if (ttlDeletion) {+ DynamoDbTtlIdentity.putOn(payload, DynamoDbTtlIdentity.Shape.EVENT_RECORD);+ } else {+ payload.putNull("userIdentity");+ }payload.put("recordFormat", "application/json");payload.put("tableName", tableName);payload.put("eventSource", "aws:dynamodb");diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.javaindex 0dd885d355..50e5b23127 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java@@ -177,6 +177,9 @@ public class NativeDynamoDbStreamsJsonHandler {node.put("eventVersion", record.getEventVersion());node.put("eventSource", record.getEventSource());node.put("awsRegion", record.getAwsRegion());+ if (record.isTtlDeletion()) {+ DynamoDbTtlIdentity.putOn(node, DynamoDbTtlIdentity.Shape.STREAMS_API);+ }ObjectNode dynamodb = objectMapper.createObjectNode();dynamodb.put("ApproximateCreationDateTime", record.getApproximateCreationDateTime());diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.javaindex 768d3d840c..d0ab1b53ca 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java@@ -17,6 +17,7 @@ import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.DynamoDbTableNames;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;@@ -53,6 +54,7 @@ public class DynamoDbLocalBackend implements DynamoDbBackend {private static final String LOCAL_ARN_PREFIX = "arn:aws:dynamodb:ddblocal:000000000000:"; // partition-literal: DynamoDB Local's fixed ARN prefixprivate static final String LOCAL_REGION = "ddblocal";+ private static final String LOCAL_TTL_EVENT_NAME = "UNKNOWN_TO_SDK_VERSION";private static final Duration READINESS_BUDGET = Duration.ofSeconds(30);private static final Scope PROBE = new Scope("000000000000", "us-east-1"); // partition-literal: readiness probe namespace, Local accepts any regionprivate static final String REPLICAS_UNSUPPORTED = "Replicas are not supported by the DynamoDB Local backend";@@ -171,9 +173,26 @@ public class DynamoDbLocalBackend implements DynamoDbBackend {}}toPublic(scope, reply.body());+ if (reply.status() >= 200 && reply.status() < 300 && call.api() == Api.DYNAMODB_STREAMS+ && "GetRecords".equals(action)) {+ markTtlDeletions(reply.body());+ }return reply;}+ // DynamoDB Local writes a TTL deletion under an event name the SDK does not know and with no identity.+ private static void markTtlDeletions(JsonNode reply) {+ if (reply == null) {+ return;+ }+ for (JsonNode record : reply.path("Records")) {+ if (record instanceof ObjectNode object && LOCAL_TTL_EVENT_NAME.equals(text(object, "eventName"))) {+ object.put("eventName", "REMOVE");+ DynamoDbTtlIdentity.putOn(object, DynamoDbTtlIdentity.Shape.STREAMS_API);+ }+ }+ }+private Reply send(Scope scope, Api api, String action, JsonNode body) {try {return client.send(scope, api, action, body);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.javaindex 0b92566045..6f1f11eebf 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java@@ -19,6 +19,7 @@ public class DynamoDbStreamRecord {private JsonNode newImage;private JsonNode oldImage;private String streamViewType;+ private boolean ttlDeletion;public DynamoDbStreamRecord() {}@@ -56,4 +57,7 @@ public class DynamoDbStreamRecord {public String getStreamViewType() { return streamViewType; }public void setStreamViewType(String streamViewType) { this.streamViewType = streamViewType; }++ public boolean isTtlDeletion() { return ttlDeletion; }+ public void setTtlDeletion(boolean ttlDeletion) { this.ttlDeletion = ttlDeletion; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 406f71bf0f..8f41eed54a 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;import io.github.hectorvent.floci.services.lambda.model.InvocationType;@@ -737,13 +738,15 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {/*** Builds the single-record node: a copy of the AWS stream record, whose top-level {@code eventName}* and metadata and {@code dynamodb} map with AttributeValue-wrapped images Lambda delivers as they- * are, plus {@code eventSourceARN}. {@code ApproximateCreationDateTime} is written as a double, as- * Floci has always delivered it. This is both the delivery record shape and the exact structure a- * DynamoDB filter pattern matches against, so it serves the matcher unchanged. (Numeric operators- * naturally never match here because AttributeValue numbers are JSON strings, AWS parity for free.)+ * are, plus {@code eventSourceARN}, with a {@code userIdentity} renamed from the Streams API's+ * capitalised members to the event record's. {@code ApproximateCreationDateTime} is written as a+ * double, as Floci has always delivered it. This is both the delivery record shape and the exact+ * structure a DynamoDB filter pattern matches against, so it serves the matcher unchanged. (Numeric+ * operators naturally never match here because AttributeValue numbers are JSON strings, AWS parity+ * for free.)*/private ObjectNode buildDynamoDbRecordNode(DynamoDbStreamReader.Record rec, EventSourceMapping esm) {- ObjectNode item = rec.awsRecord().deepCopy();+ ObjectNode item = DynamoDbTtlIdentity.toEventRecord(rec.awsRecord());item.put("eventSourceARN", esm.getEventSourceArn());JsonNode created = item.path("dynamodb").path("ApproximateCreationDateTime");if (created.isNumber()) {diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.javaindex 4352ad12e0..95e6e9e8ac 100644--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;import io.github.hectorvent.floci.services.kinesis.KinesisService;import io.github.hectorvent.floci.services.pipes.model.DesiredState;@@ -365,7 +366,7 @@ public class PipesPoller implements Resettable {LOG.infov("Pipe {0}: received {1} DynamoDB Stream record(s)", pipe.getName(), records.size());List<JsonNode> recordNodes = new ArrayList<>(records.size());for (DynamoDbStreamReader.Record record : records) {- ObjectNode node = record.awsRecord().deepCopy();+ ObjectNode node = DynamoDbTtlIdentity.toEventRecord(record.awsRecord());node.put("eventSourceARN", pipe.getSource());recordNodes.add(node);}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.javaindex fc6c02fb37..7e35745b41 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java@@ -13,7 +13,10 @@ import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;import java.time.Duration;+import java.time.Instant;+import java.util.ArrayList;import java.util.Base64;+import java.util.List;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;@@ -32,6 +35,9 @@ class DynamoDbKinesisStreamingIntegrationTest {@InjectKinesisStreamingForwarder forwarder;+ @Inject+ DynamoDbService dynamoDbService;+@BeforeAllstatic void configureRestAssured() {RestAssuredJsonUtils.configureAwsContentTypes();@@ -293,7 +299,7 @@ class DynamoDbKinesisStreamingIntegrationTest {@Test@Order(12)- void deleteItemForwardsRemoveEvent() {+ void deleteItemForwardsRemoveEvent() throws Exception {given().header("X-Amz-Target", "DynamoDB_20120810.DeleteItem").contentType(DYNAMODB_CONTENT_TYPE)@@ -334,6 +340,8 @@ class DynamoDbKinesisStreamingIntegrationTest {String lastEncoded = recordsResponse.jsonPath().getString("Records[" + (recordCount - 1) + "].Data");String lastDecoded = new String(Base64.getDecoder().decode(lastEncoded));assertTrue(lastDecoded.contains("\"eventName\":\"REMOVE\""), "Expected REMOVE event, got: " + lastDecoded);+ assertTrue(new ObjectMapper().readTree(lastDecoded).get("userIdentity").isNull(),+ "a DeleteItem removal is not made by the DynamoDB service: " + lastDecoded);}@Test@@ -481,6 +489,77 @@ class DynamoDbKinesisStreamingIntegrationTest {.body("DestinationStatus", equalTo("ACTIVE"));}+ @Test+ @Order(31)+ void ttlDeletionForwardsTheDynamoDbServiceIdentity() throws Exception {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTimeToLive")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "StreamingTable",+ "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "expireAt"}+ }+ """)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ long past = Instant.now().getEpochSecond() - 3600;+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableName\": \"StreamingTable\", \"Item\": {\"pk\": {\"S\": \"expired\"}, "+ + "\"expireAt\": {\"N\": \"" + past + "\"}}}")+ .when()+ .post("/")+ .then()+ .statusCode(200);++ dynamoDbService.deleteExpiredItems();+ awaitCdcDelivery();++ JsonNode removal = null;+ for (JsonNode payload : kinesisPayloads()) {+ if ("REMOVE".equals(payload.path("eventName").asText())+ && "expired".equals(payload.path("dynamodb").path("Keys").path("pk").path("S").asText())) {+ removal = payload;+ }+ }+ assertNotNull(removal, "the TTL deletion must be forwarded to Kinesis");+ assertEquals(new ObjectMapper().readTree("{\"type\":\"Service\",\"principalId\":\"dynamodb.amazonaws.com\"}"),+ removal.get("userIdentity"));+ }++ private List<JsonNode> kinesisPayloads() throws Exception {+ String shardIterator = given()+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("""+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}+ """)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().jsonPath().getString("ShardIterator");+ List<String> encoded = given()+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")+ .contentType(KINESIS_CONTENT_TYPE)+ .body("{\"ShardIterator\": \"" + shardIterator + "\", \"Limit\": 100}")+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().jsonPath().getList("Records.Data", String.class);+ ObjectMapper mapper = new ObjectMapper();+ List<JsonNode> payloads = new ArrayList<>();+ for (String data : encoded) {+ payloads.add(mapper.readTree(Base64.getDecoder().decode(data)));+ }+ return payloads;+ }+@Test@Order(40)void enableLeavesDynamoDbStreamsDisabled() {diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.javaindex 73ce06ab5a..2bf8f0799b 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java@@ -398,4 +398,35 @@ class DynamoDbStreamServiceTest {assertEquals(1, service.listStreams("ViewTypeTable", "000000000000", "us-east-1").size());}+ @Test+ void captureEvent_ttlRemoval_isMarkedAsATtlDeletion() throws Exception {+ TableDefinition table = sharedNameTable(ACCOUNT_A);+ StreamDescription stream = service.enableStream(SHARED_TABLE, table.getTableArn(), "OLD_IMAGE", "us-east-1");++ service.captureEvent("REMOVE", mapper.readTree("{\"userId\":{\"S\":\"a\"}}"), null, table, "us-east-1", true);++ DynamoDbStreamRecord record = onlyRecordIn(stream.getStreamArn());+ assertEquals("REMOVE", record.getEventName());+ assertTrue(record.isTtlDeletion());+ }++ @Test+ void captureEvent_deleteItemRemoval_isNotATtlDeletion() throws Exception {+ TableDefinition table = sharedNameTable(ACCOUNT_A);+ StreamDescription stream = service.enableStream(SHARED_TABLE, table.getTableArn(), "OLD_IMAGE", "us-east-1");++ service.captureEvent("REMOVE", mapper.readTree("{\"userId\":{\"S\":\"a\"}}"), null, table, "us-east-1");++ DynamoDbStreamRecord record = onlyRecordIn(stream.getStreamArn());+ assertEquals("REMOVE", record.getEventName());+ assertFalse(record.isTtlDeletion());+ }++ private DynamoDbStreamRecord onlyRecordIn(String streamArn) {+ String iterator = service.getShardIterator(streamArn, DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);+ List<DynamoDbStreamRecord> records = service.getRecords(iterator, 10).records();+ assertEquals(1, records.size());+ return records.get(0);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.javaindex 7de9fc8cfb..500b125f4c 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java@@ -61,6 +61,11 @@ class DynamoDbTtlForwardingTest {};private static final String REGION = "us-east-1";+ private static final String KINESIS_TTL_IDENTITY =+ "{\"type\":\"Service\",\"principalId\":\"dynamodb.amazonaws.com\"}";+ /** The DynamoDB Streams API capitalises the Identity members, as real AWS returns them. */+ private static final String STREAMS_TTL_IDENTITY =+ "{\"Type\":\"Service\",\"PrincipalId\":\"dynamodb.amazonaws.com\"}";private final ObjectMapper mapper = new ObjectMapper();@@ -141,6 +146,8 @@ class DynamoDbTtlForwardingTest {for (KinesisRecord record : drained) {JsonNode payload = mapper.readTree(record.getData());assertEquals("REMOVE", payload.get("eventName").asText(), "TTL deletions forward as REMOVE");+ assertEquals(mapper.readTree(KINESIS_TTL_IDENTITY), payload.get("userIdentity"),+ "TTL deletions forward as made by the DynamoDB service");}assertEquals(0L, forwarder.getForwardFailureCount(), "happy path drops nothing");@@ -218,7 +225,7 @@ class DynamoDbTtlForwardingTest {doAnswer(invocation -> {svc.deleteTable("TtlTable", REGION);return null;- }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());+ }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));svc.deleteExpiredItems();@@ -277,11 +284,11 @@ class DynamoDbTtlForwardingTest {doAnswer(invocation -> {svc.deleteTable("TtlTable", REGION);return null;- }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());+ }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));svc.deleteScannedItems(scans);- verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any());+ verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));}@Test@@ -393,4 +400,58 @@ class DynamoDbTtlForwardingTest {assertEquals(1, kinesisRemoves.size(), "a skipped delete must not forward a Kinesis REMOVE record");assertEquals("row-1", kinesisRemoves.get(0).get("dynamodb").get("Keys").get("pk").get("S").asText());}++ @Test+ void getRecords_ttlSweepRemoval_carriesTheDynamoDbServiceIdentity() throws Exception {+ StreamedTtlTable streamed = streamedTtlTable();+ streamed.service().deleteExpiredItems();++ JsonNode record = onlyRecord(streamed);+ assertEquals("REMOVE", record.path("eventName").asText());+ assertEquals("row-0", record.path("dynamodb").path("Keys").path("pk").path("S").asText());+ assertEquals(mapper.readTree(STREAMS_TTL_IDENTITY), record.path("userIdentity"));+ }++ @Test+ void getRecords_deleteItemRemoval_carriesNoIdentity() throws Exception {+ StreamedTtlTable streamed = streamedTtlTable();+ ObjectNode key = mapper.createObjectNode();+ key.set("pk", stringAttr("row-0"));+ streamed.service().deleteItem("TtlTable", key, REGION);++ JsonNode record = onlyRecord(streamed);+ assertEquals("REMOVE", record.path("eventName").asText());+ assertFalse(record.has("userIdentity"), "a DeleteItem removal is not made by the DynamoDB service");+ }++ private record StreamedTtlTable(DynamoDbService service, NativeDynamoDbStreamsJsonHandler handler,+ String streamArn) {}++ /** A TTL table holding one expired item, whose stream starts after the item was written. */+ private StreamedTtlTable streamedTtlTable() {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ DynamoDbStreamService streamService = new DynamoDbStreamService(mapper, tableStore);+ RegionResolver regionResolver = new RegionResolver(REGION, "000000000000");+ DynamoDbService svc = new DynamoDbService(+ tableStore, new InMemoryStorage<>(), regionResolver, streamService, null);+ seedTtlTable(svc, 1);+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();+ StreamDescription sd = streamService.enableStream(+ table.getTableName(), table.getTableArn(), "NEW_AND_OLD_IMAGES", REGION);+ return new StreamedTtlTable(svc,+ new NativeDynamoDbStreamsJsonHandler(streamService, svc, regionResolver, mapper), sd.getStreamArn());+ }++ private JsonNode onlyRecord(StreamedTtlTable streamed) {+ ObjectNode iteratorRequest = mapper.createObjectNode()+ .put("StreamArn", streamed.streamArn())+ .put("ShardId", DynamoDbStreamService.SHARD_ID)+ .put("ShardIteratorType", "TRIM_HORIZON");+ JsonNode iterator = (JsonNode) streamed.handler().handle("GetShardIterator", iteratorRequest, REGION).getEntity();+ ObjectNode recordsRequest = mapper.createObjectNode().put("ShardIterator", iterator.path("ShardIterator").asText());+ JsonNode records = ((JsonNode) streamed.handler().handle("GetRecords", recordsRequest, REGION).getEntity())+ .path("Records");+ assertEquals(1, records.size(), "one stream record: " + records);+ return records.get(0);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.javaindex c85c2ac86a..25835b1f08 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java@@ -20,6 +20,8 @@ import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;+import java.time.Duration;+import java.time.Instant;import java.util.ArrayList;import java.util.List;import java.util.Map;@@ -27,6 +29,7 @@ import java.util.UUID;import java.util.concurrent.ThreadLocalRandom;import static io.restassured.RestAssured.given;+import static org.awaitility.Awaitility.await;import static org.hamcrest.Matchers.contains;import static org.hamcrest.Matchers.containsInAnyOrder;import static org.hamcrest.Matchers.endsWith;@@ -203,6 +206,38 @@ class DynamoDbLocalBackendIntegrationTest {assertEquals(CheckpointLifetime.STREAM, streamReader.checkpointLifetime());}+ @Test+ void aTtlDeletionReadsAsRemoveByTheDynamoDbService() {+ String table = tableName("ttl");+ String streamArn = createTable(A_WEST, table, STREAMED)+ .statusCode(200)+ .extract().path("TableDescription.LatestStreamArn");+ dynamoDb(A_WEST, "UpdateTimeToLive", "{\"TableName\": \"" + table+ + "\", \"TimeToLiveSpecification\": {\"Enabled\": true, \"AttributeName\": \"expireAt\"}}")+ .statusCode(200);+ long past = Instant.now().getEpochSecond() - 3600;+ dynamoDb(A_WEST, "PutItem", "{\"TableName\": \"" + table + "\", \"Item\": {\"pk\": {\"S\": \"expired\"}, "+ + "\"expireAt\": {\"N\": \"" + past + "\"}}}")+ .statusCode(200);+ String shardId = streams(A_WEST, "DescribeStream", "{\"StreamArn\": \"" + streamArn + "\"}")+ .statusCode(200)+ .extract().path("StreamDescription.Shards[0].ShardId");++ // DynamoDB Local deleted an expired item within about 15 seconds when measured.+ await().atMost(Duration.ofSeconds(60)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> {+ String iterator = streams(A_WEST, "GetShardIterator", "{\"StreamArn\": \"" + streamArn+ + "\", \"ShardId\": \"" + shardId + "\", \"ShardIteratorType\": \"TRIM_HORIZON\"}")+ .statusCode(200)+ .extract().path("ShardIterator");+ streams(A_WEST, "GetRecords", "{\"ShardIterator\": \"" + iterator + "\"}")+ .statusCode(200)+ .body("Records.eventName", contains("INSERT", "REMOVE"))+ .body("Records[1].dynamodb.Keys.pk.S", equalTo("expired"))+ .body("Records[1].userIdentity.Type", equalTo("Service"))+ .body("Records[1].userIdentity.PrincipalId", equalTo("dynamodb.amazonaws.com"));+ });+ }+@Testvoid theDirectDeleteRemovesAProtectedTable() {String table = tableName("protected");diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.javaindex 9c2f2dc57d..3d64822d74 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java@@ -278,6 +278,32 @@ class DynamoDbLocalBackendTest {assertEquals(REGION, reply.body().path("Records").path(0).path("awsRegion").asText());}+ @Test+ void timeToLiveDeletionReadsAsRemoveByTheDynamoDbService() throws Exception {+ answer(Api.DYNAMODB_STREAMS, "GetRecords", 200, "{\"Records\":[{\"eventName\":\"UNKNOWN_TO_SDK_VERSION\","+ + "\"awsRegion\":\"ddblocal\",\"dynamodb\":{\"Keys\":{\"pk\":{\"S\":\"a\"}},"+ + "\"OldImage\":{\"pk\":{\"S\":\"a\"}}}}]}");++ Reply reply = execute(Api.DYNAMODB_STREAMS, "GetRecords", "{\"ShardIterator\":\"it\"}");++ JsonNode record = reply.body().path("Records").path(0);+ assertEquals("REMOVE", record.path("eventName").asText());+ assertEquals(json("{\"Type\":\"Service\",\"PrincipalId\":\"dynamodb.amazonaws.com\"}"),+ record.path("userIdentity"));+ assertEquals("a", record.path("dynamodb").path("OldImage").path("pk").path("S").asText());+ }++ @Test+ void writeRecordsKeepTheirEventNameAndCarryNoIdentity() throws Exception {+ String records = "[{\"eventName\":\"INSERT\",\"dynamodb\":{}},{\"eventName\":\"MODIFY\",\"dynamodb\":{}},"+ + "{\"eventName\":\"REMOVE\",\"dynamodb\":{}}]";+ answer(Api.DYNAMODB_STREAMS, "GetRecords", 200, "{\"Records\":" + records + "}");++ Reply reply = execute(Api.DYNAMODB_STREAMS, "GetRecords", "{\"ShardIterator\":\"it\"}");++ assertEquals(json(records), reply.body().path("Records"));+ }+@Testvoid arnLookingItemAttributeUntouched() throws Exception {String item = "{\"TableArn\":{\"S\":\"" + LOCAL + "table/x\"},"diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex 1e3e4f54f6..a37622624b 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -543,6 +543,45 @@ class DynamoDbStreamsEventSourcePollerTest {assertEquals("s2", esm.getShardSequenceNumbers().get(SHARD_ID));}+ @Test+ void userIdentityFilterDeliversOnlyTheTtlDeletion() {+ stubTrimHorizon(List.of(+ ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}"),+ removal("s2", false),+ removal("s3", true)));+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(new InvokeResult());+ EventSourceMapping esm = filterEsm(+ "{\"userIdentity\":{\"type\":[\"Service\"],\"principalId\":[\"dynamodb.amazonaws.com\"]}}");++ pollerWith(mock(EsmStore.class)).pollAndInvoke(esm);++ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));+ JsonNode records = readRecords(payload.getValue());+ assertEquals(1, records.size());+ assertEquals("s3", records.get(0).path("dynamodb").path("SequenceNumber").asText());+ assertEquals("REMOVE", records.get(0).path("eventName").asText());+ assertEquals(OBJECT_MAPPER.createObjectNode().put("type", "Service").put("principalId", "dynamodb.amazonaws.com"),+ records.get(0).path("userIdentity"));+ }++ /**+ * A REMOVE record as GetRecords returns it. One that time to live made carries the DynamoDB service+ * identity with the Streams API's capitalised member names, as AWS writes it.+ */+ private DynamoDbStreamReader.Record removal(String seq, boolean byTtl) {+ DynamoDbStreamReader.Record record = ddbRecord(seq, "REMOVE", "{\"status\":{\"S\":\"expired\"}}");+ ObjectNode dynamodb = (ObjectNode) record.awsRecord().path("dynamodb");+ dynamodb.set("OldImage", dynamodb.remove("NewImage"));+ if (byTtl) {+ ((ObjectNode) record.awsRecord()).putObject("userIdentity")+ .put("PrincipalId", "dynamodb.amazonaws.com")+ .put("Type", "Service");+ }+ return record;+ }+@Testvoid partialBatchFailureCheckpointsBeforeLowestFailedRecord() {stubTrimHorizon(List.of(diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.javaindex beff9ac091..9c9ff83c69 100644--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java@@ -549,6 +549,29 @@ class PipesPollerTest {assertEquals(original, active.awsRecord(), "the reader's record is never mutated");}+ @Test+ void userIdentityFilterDeliversOnlyTheTtlDeletionWithTheEventRecordIdentity() throws Exception {+ Pipe pipe = dynamoDbPipe("{}");+ pipe.setSourceParameters(MAPPER.readTree("""+ {"DynamoDBStreamParameters":{"StartingPosition":"TRIM_HORIZON"},+ "FilterCriteria":{"Filters":[{"Pattern":+ "{\\"userIdentity\\":{\\"type\\":[\\"Service\\"],\\"principalId\\":[\\"dynamodb.amazonaws.com\\"]}}"}]}}"""));+ streams.shard.add(ddbRemoval("001", false));+ DynamoDbStreamReader.Record ttlRemoval = ddbRemoval("002", true);+ JsonNode original = ttlRemoval.awsRecord().deepCopy();+ streams.shard.add(ttlRemoval);++ poller.startPolling(pipe);+ poller.pollDynamoDbStreams(pipe, "us-east-1");++ assertEquals(List.of(List.of("002")), deliveredSequences(pipe));+ ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);+ verify(targetInvoker).invoke(eq(pipe), payload.capture(), eq("us-east-1"));+ assertEquals(MAPPER.createObjectNode().put("type", "Service").put("principalId", "dynamodb.amazonaws.com"),+ MAPPER.readTree(payload.getValue()).path("Records").path(0).path("userIdentity"));+ assertEquals(original, ttlRemoval.awsRecord(), "the reader's record is never mutated");+ }+@Testvoid trimmedDynamoDbCheckpointRestartsAtTheTrimHorizon() throws Exception {Pipe pipe = dynamoDbPipe("{}");@@ -641,6 +664,22 @@ class PipesPollerTest {return new DynamoDbStreamReader.Record(sequenceNumber, awsRecord);}+ /**+ * A REMOVE record as GetRecords returns it. One that time to live made carries the DynamoDB service+ * identity with the Streams API's capitalised member names, as AWS writes it.+ */+ private static DynamoDbStreamReader.Record ddbRemoval(String sequenceNumber, boolean byTtl) {+ DynamoDbStreamReader.Record record = ddbRecord(sequenceNumber, "expired");+ ObjectNode awsRecord = (ObjectNode) record.awsRecord();+ awsRecord.put("eventName", "REMOVE");+ ObjectNode dynamodb = (ObjectNode) awsRecord.path("dynamodb");+ dynamodb.set("OldImage", dynamodb.remove("NewImage"));+ if (byTtl) {+ awsRecord.putObject("userIdentity").put("PrincipalId", "dynamodb.amazonaws.com").put("Type", "Service");+ }+ return record;+ }+/** The sequence numbers of each batch the Lambda target was invoked with, in invocation order. */private List<List<String>> deliveredSequences(Pipe pipe) throws Exception {ArgumentCaptor<String> payloads = ArgumentCaptor.forClass(String.class);