Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): mark time-to-live deletions on the stream as AWS does (#4930)

Mario Toffia9a7ec1c9d0b8
diff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
index 4a6cb9ec37..a19956d461 100644
--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java
@@ -49,6 +49,7 @@ import software.amazon.awssdk.services.datasync.DataSyncClient;
import software.amazon.awssdk.services.detective.DetectiveClient;
import software.amazon.awssdk.services.dlm.DlmClient;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClient;
import software.amazon.awssdk.services.ec2.Ec2Client;
import software.amazon.awssdk.services.ecr.EcrClient;
import software.amazon.awssdk.services.ecs.EcsClient;
@@ -713,6 +714,14 @@ public final class TestFixtures {
.build();
}
+ public static DynamoDbStreamsClient dynamoDbStreamsClient() {
+ return DynamoDbStreamsClient.builder()
+ .endpointOverride(ENDPOINT)
+ .region(REGION)
+ .credentialsProvider(CREDENTIALS)
+ .build();
+ }
+
public static LambdaClient lambdaClient() {
return LambdaClient.builder()
.endpointOverride(ENDPOINT)
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.java
new file mode 100644
index 0000000000..7f96a30e23
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbStreamsTest.java
@@ -0,0 +1,110 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition;
+import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
+import software.amazon.awssdk.services.dynamodb.model.BillingMode;
+import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest;
+import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest;
+import software.amazon.awssdk.services.dynamodb.model.DescribeStreamRequest;
+import software.amazon.awssdk.services.dynamodb.model.GetRecordsRequest;
+import software.amazon.awssdk.services.dynamodb.model.GetShardIteratorRequest;
+import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement;
+import software.amazon.awssdk.services.dynamodb.model.KeyType;
+import software.amazon.awssdk.services.dynamodb.model.PutItemRequest;
+import software.amazon.awssdk.services.dynamodb.model.Record;
+import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException;
+import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType;
+import software.amazon.awssdk.services.dynamodb.model.Shard;
+import software.amazon.awssdk.services.dynamodb.model.ShardIteratorType;
+import software.amazon.awssdk.services.dynamodb.model.StreamSpecification;
+import software.amazon.awssdk.services.dynamodb.model.StreamViewType;
+import software.amazon.awssdk.services.dynamodb.streams.DynamoDbStreamsClient;
+
+import java.util.Map;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Reads stream records through the SDK's DynamoDB Streams client, whose JSON unmarshaller matches the
+ * Streams API's capitalised members exactly.
+ */
+@DisplayName("DynamoDB Streams")
+class DynamoDbStreamsTest {
+
+ private static final String TABLE_NAME = TestFixtures.uniqueName("sdk-stream");
+
+ private static DynamoDbClient ddb;
+ private static DynamoDbStreamsClient streams;
+ private static String streamArn;
+
+ @BeforeAll
+ static void setup() {
+ ddb = TestFixtures.dynamoDbClient();
+ streams = TestFixtures.dynamoDbStreamsClient();
+ streamArn = ddb.createTable(CreateTableRequest.builder()
+ .tableName(TABLE_NAME)
+ .keySchema(KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build())
+ .attributeDefinitions(AttributeDefinition.builder()
+ .attributeName("pk").attributeType(ScalarAttributeType.S).build())
+ .billingMode(BillingMode.PAY_PER_REQUEST)
+ .streamSpecification(StreamSpecification.builder()
+ .streamEnabled(true).streamViewType(StreamViewType.NEW_AND_OLD_IMAGES).build())
+ .build())
+ .tableDescription().latestStreamArn();
+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
+ .item(Map.of("pk", AttributeValue.fromS("deleted")))
+ .build());
+ ddb.deleteItem(DeleteItemRequest.builder().tableName(TABLE_NAME)
+ .key(Map.of("pk", AttributeValue.fromS("deleted")))
+ .build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ if (ddb != null) {
+ try {
+ ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());
+ } catch (ResourceNotFoundException ignored) {
+ // The table is gone already when setup failed before creating it.
+ }
+ ddb.close();
+ }
+ if (streams != null) {
+ streams.close();
+ }
+ }
+
+ @Test
+ @DisplayName("GetRecords returns a DeleteItem removal with no identity")
+ void deleteItemRemovalCarriesNoIdentity() {
+ Optional<Record> removal = findRemovalOf("deleted");
+
+ assertThat(removal).hasValueSatisfying(record -> assertThat(record.userIdentity()).isNull());
+ }
+
+ private static Optional<Record> findRemovalOf(String pk) {
+ for (Shard shard : streams.describeStream(DescribeStreamRequest.builder().streamArn(streamArn).build())
+ .streamDescription().shards()) {
+ String iterator = streams.getShardIterator(GetShardIteratorRequest.builder()
+ .streamArn(streamArn)
+ .shardId(shard.shardId())
+ .shardIteratorType(ShardIteratorType.TRIM_HORIZON)
+ .build()).shardIterator();
+ for (Record record : streams.getRecords(GetRecordsRequest.builder().shardIterator(iterator).build())
+ .records()) {
+ if ("REMOVE".equals(record.eventNameAsString())
+ && pk.equals(record.dynamodb().keys().get("pk").s())) {
+ return Optional.of(record);
+ }
+ }
+ }
+ return Optional.empty();
+ }
+}
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 5b25fec9b8..c68030c9a6 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -70,6 +70,26 @@ epoch time in the past. Each deletion writes a `REMOVE` record to the table's st
forwarded to an active Kinesis streaming destination. Between sweeps, reads already leave expired
items out.
+As in AWS, the records of a TTL deletion carry the DynamoDB service identity, which a `DeleteItem`
+removal does not. The member names depend on where the record is read. The DynamoDB Streams API
+(`GetRecords`) capitalises them, as the SDK's `Identity` shape expects:
+
+```json
+"userIdentity": {"PrincipalId": "dynamodb.amazonaws.com", "Type": "Service"}
+```
+
+The records that Lambda event source mappings and EventBridge Pipes deliver and filter use the
+lowercase names the DynamoDB developer guide shows, and so does the Kinesis streaming destination
+record:
+
+```json
+"userIdentity": {"type": "Service", "principalId": "dynamodb.amazonaws.com"}
+```
+
+A Lambda event source mapping or an EventBridge pipe that should see only TTL deletions filters on
+it with the pattern
+`{"userIdentity": {"type": ["Service"], "principalId": ["dynamodb.amazonaws.com"]}}`.
+
## DynamoDB Local backend
By default Floci stores tables in its own engine, the `native` backend. With
@@ -128,6 +148,11 @@ region is rejected.
`memory` storage they are lost when Floci restarts while the tables stay in DynamoDB Local.
- **Stream consumers.** Lambda event source mappings, EventBridge Pipes and other Floci stream
consumers read the DynamoDB Local streams through the Streams API.
+- **Time to live deletions.** DynamoDB Local deletes expired items itself and writes each deletion
+ to the stream with the event name `UNKNOWN_TO_SDK_VERSION` and no `userIdentity`. Floci's
+ `GetRecords` returns it as AWS writes it, a `REMOVE` record with the DynamoDB service
+ `userIdentity` (see [Time to Live](#time-to-live)), so every stream consumer and event filter
+ sees the AWS shape.
### Limits
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index c95b93aca0..879511d66c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -2778,12 +2778,13 @@ public class DynamoDbService {
}
deletedForTable++;
if (streamService != null) {
- streamService.captureEvent("REMOVE", removed, null, scan.table(), scan.region());
+ streamService.captureEvent("REMOVE", removed, null, scan.table(), scan.region(), true);
}
if (kinesisForwarder != null) {
// Out of request scope here: pass the table owner's account explicitly so the CDC
// record lands in the owner's stream, not the default account's same-named stream.
- kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId());
+ kinesisForwarder.forward("REMOVE", removed, null, scan.table(), scan.region(), scan.accountId(),
+ true);
}
}
if (deletedForTable > 0) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index 023737132e..5970c67750 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -157,6 +157,12 @@ public class DynamoDbStreamService {
public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,
TableDefinition table, String region) {
+ captureEvent(eventName, oldItem, newItem, table, region, false);
+ }
+
+ /** {@code ttlDeletion} marks a removal made by the time to live sweep rather than by a caller. */
+ public void captureEvent(String eventName, JsonNode oldItem, JsonNode newItem,
+ TableDefinition table, String region, boolean ttlDeletion) {
StreamDescription sd = streams.get(table.getTableArn());
if (sd == null || !"ENABLED".equals(sd.getStreamStatus())) {
return;
@@ -184,6 +190,7 @@ public class DynamoDbStreamService {
record.setNewImage(newImage);
record.setOldImage(oldImage);
record.setStreamViewType(viewType);
+ record.setTtlDeletion(ttlDeletion);
ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(sd.getStreamArn());
if (deque != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.java
new file mode 100644
index 0000000000..d8b84ca6d4
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlIdentity.java
@@ -0,0 +1,54 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.ServicePrincipals;
+
+/**
+ * The {@code userIdentity} AWS writes on the records of an item that time to live deleted. A consumer
+ * tells a TTL deletion from a {@code DeleteItem} by it. Each output names the identity's members its
+ * own way, and {@link Shape} holds the names of every output.
+ */
+public final class DynamoDbTtlIdentity {
+
+ private static final String USER_IDENTITY = "userIdentity";
+
+ /** An output a record is read from, with the member names its identity has there. */
+ public enum Shape {
+ /** The DynamoDB Streams API, whose {@code Identity} members AWS returns capitalised. */
+ STREAMS_API("Type", "PrincipalId"),
+ /**
+ * The Lambda and EventBridge Pipes event record, and the Kinesis Data Streams destination
+ * payload: lowercase names, as measured against AWS.
+ */
+ EVENT_RECORD("type", "principalId");
+
+ private final String type;
+ private final String principalId;
+
+ Shape(String type, String principalId) {
+ this.type = type;
+ this.principalId = principalId;
+ }
+ }
+
+ private DynamoDbTtlIdentity() {
+ }
+
+ public static void putOn(ObjectNode record, Shape shape) {
+ record.putObject(USER_IDENTITY)
+ .put(shape.type, "Service")
+ .put(shape.principalId, ServicePrincipals.of("dynamodb"));
+ }
+
+ /** A copy of a Streams API record, its identity renamed to the event record shape Lambda and Pipes deliver. */
+ public static ObjectNode toEventRecord(JsonNode streamsRecord) {
+ ObjectNode record = streamsRecord.deepCopy();
+ if (record.get(USER_IDENTITY) instanceof ObjectNode identity) {
+ ObjectNode event = record.putObject(USER_IDENTITY);
+ event.set(Shape.EVENT_RECORD.type, identity.get(Shape.STREAMS_API.type));
+ event.set(Shape.EVENT_RECORD.principalId, identity.get(Shape.STREAMS_API.principalId));
+ }
+ return record;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
index 8632558ddb..506ce16dad 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
@@ -140,6 +140,12 @@ public class KinesisStreamingForwarder {
public void forward(String eventName, JsonNode oldItem, JsonNode newItem,
TableDefinition table, String region, String ownerAccountId) {
+ forward(eventName, oldItem, newItem, table, region, ownerAccountId, false);
+ }
+
+ /** {@code ttlDeletion} marks a removal made by the time to live sweep rather than by a caller. */
+ public void forward(String eventName, JsonNode oldItem, JsonNode newItem,
+ TableDefinition table, String region, String ownerAccountId, boolean ttlDeletion) {
List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();
if (destinations == null || destinations.isEmpty()) {
return;
@@ -169,7 +175,7 @@ public class KinesisStreamingForwarder {
for (String precision : precisionByStreamArn.values()) {
if (!dataByPrecision.containsKey(precision)) {
ObjectNode payload = buildPayload(eventId, eventName, keys, newItem, oldItem,
- table.getTableName(), region, now, precision);
+ table.getTableName(), region, now, precision, ttlDeletion);
dataByPrecision.put(precision, objectMapper.writeValueAsBytes(payload));
}
}
@@ -441,12 +447,17 @@ public class KinesisStreamingForwarder {
private ObjectNode buildPayload(String eventId, String eventName, JsonNode keys,
JsonNode newImage, JsonNode oldImage,
- String tableName, String region, Instant timestamp, String precision) {
+ String tableName, String region, Instant timestamp, String precision,
+ boolean ttlDeletion) {
ObjectNode payload = objectMapper.createObjectNode();
payload.put("awsRegion", region);
payload.put("eventID", eventId);
payload.put("eventName", eventName);
- payload.putNull("userIdentity");
+ if (ttlDeletion) {
+ DynamoDbTtlIdentity.putOn(payload, DynamoDbTtlIdentity.Shape.EVENT_RECORD);
+ } else {
+ payload.putNull("userIdentity");
+ }
payload.put("recordFormat", "application/json");
payload.put("tableName", tableName);
payload.put("eventSource", "aws:dynamodb");
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java
index 0dd885d355..50e5b23127 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbStreamsJsonHandler.java
@@ -177,6 +177,9 @@ public class NativeDynamoDbStreamsJsonHandler {
node.put("eventVersion", record.getEventVersion());
node.put("eventSource", record.getEventSource());
node.put("awsRegion", record.getAwsRegion());
+ if (record.isTtlDeletion()) {
+ DynamoDbTtlIdentity.putOn(node, DynamoDbTtlIdentity.Shape.STREAMS_API);
+ }
ObjectNode dynamodb = objectMapper.createObjectNode();
dynamodb.put("ApproximateCreationDateTime", record.getApproximateCreationDateTime());
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java
index 768d3d840c..d0ab1b53ca 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackend.java
@@ -17,6 +17,7 @@ import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbTableNames;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
@@ -53,6 +54,7 @@ public class DynamoDbLocalBackend implements DynamoDbBackend {
private static final String LOCAL_ARN_PREFIX = "arn:aws:dynamodb:ddblocal:000000000000:"; // partition-literal: DynamoDB Local's fixed ARN prefix
private static final String LOCAL_REGION = "ddblocal";
+ private static final String LOCAL_TTL_EVENT_NAME = "UNKNOWN_TO_SDK_VERSION";
private static final Duration READINESS_BUDGET = Duration.ofSeconds(30);
private static final Scope PROBE = new Scope("000000000000", "us-east-1"); // partition-literal: readiness probe namespace, Local accepts any region
private static final String REPLICAS_UNSUPPORTED = "Replicas are not supported by the DynamoDB Local backend";
@@ -171,9 +173,26 @@ public class DynamoDbLocalBackend implements DynamoDbBackend {
}
}
toPublic(scope, reply.body());
+ if (reply.status() >= 200 && reply.status() < 300 && call.api() == Api.DYNAMODB_STREAMS
+ && "GetRecords".equals(action)) {
+ markTtlDeletions(reply.body());
+ }
return reply;
}
+ // DynamoDB Local writes a TTL deletion under an event name the SDK does not know and with no identity.
+ private static void markTtlDeletions(JsonNode reply) {
+ if (reply == null) {
+ return;
+ }
+ for (JsonNode record : reply.path("Records")) {
+ if (record instanceof ObjectNode object && LOCAL_TTL_EVENT_NAME.equals(text(object, "eventName"))) {
+ object.put("eventName", "REMOVE");
+ DynamoDbTtlIdentity.putOn(object, DynamoDbTtlIdentity.Shape.STREAMS_API);
+ }
+ }
+ }
+
private Reply send(Scope scope, Api api, String action, JsonNode body) {
try {
return client.send(scope, api, action, body);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java
index 0b92566045..6f1f11eebf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/DynamoDbStreamRecord.java
@@ -19,6 +19,7 @@ public class DynamoDbStreamRecord {
private JsonNode newImage;
private JsonNode oldImage;
private String streamViewType;
+ private boolean ttlDeletion;
public DynamoDbStreamRecord() {}
@@ -56,4 +57,7 @@ public class DynamoDbStreamRecord {
public String getStreamViewType() { return streamViewType; }
public void setStreamViewType(String streamViewType) { this.streamViewType = streamViewType; }
+
+ public boolean isTtlDeletion() { return ttlDeletion; }
+ public void setTtlDeletion(boolean ttlDeletion) { this.ttlDeletion = ttlDeletion; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 406f71bf0f..8f41eed54a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
@@ -737,13 +738,15 @@ public class DynamoDbStreamsEventSourcePoller implements Resettable {
/**
* Builds the single-record node: a copy of the AWS stream record, whose top-level {@code eventName}
* and metadata and {@code dynamodb} map with AttributeValue-wrapped images Lambda delivers as they
- * are, plus {@code eventSourceARN}. {@code ApproximateCreationDateTime} is written as a double, as
- * Floci has always delivered it. This is both the delivery record shape and the exact structure a
- * DynamoDB filter pattern matches against, so it serves the matcher unchanged. (Numeric operators
- * naturally never match here because AttributeValue numbers are JSON strings, AWS parity for free.)
+ * are, plus {@code eventSourceARN}, with a {@code userIdentity} renamed from the Streams API's
+ * capitalised members to the event record's. {@code ApproximateCreationDateTime} is written as a
+ * double, as Floci has always delivered it. This is both the delivery record shape and the exact
+ * structure a DynamoDB filter pattern matches against, so it serves the matcher unchanged. (Numeric
+ * operators naturally never match here because AttributeValue numbers are JSON strings, AWS parity
+ * for free.)
*/
private ObjectNode buildDynamoDbRecordNode(DynamoDbStreamReader.Record rec, EventSourceMapping esm) {
- ObjectNode item = rec.awsRecord().deepCopy();
+ ObjectNode item = DynamoDbTtlIdentity.toEventRecord(rec.awsRecord());
item.put("eventSourceARN", esm.getEventSourceArn());
JsonNode created = item.path("dynamodb").path("ApproximateCreationDateTime");
if (created.isNumber()) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
index 4352ad12e0..95e6e9e8ac 100644
--- a/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/pipes/PipesPoller.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Resettable;
+import io.github.hectorvent.floci.services.dynamodb.DynamoDbTtlIdentity;
import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.pipes.model.DesiredState;
@@ -365,7 +366,7 @@ public class PipesPoller implements Resettable {
LOG.infov("Pipe {0}: received {1} DynamoDB Stream record(s)", pipe.getName(), records.size());
List<JsonNode> recordNodes = new ArrayList<>(records.size());
for (DynamoDbStreamReader.Record record : records) {
- ObjectNode node = record.awsRecord().deepCopy();
+ ObjectNode node = DynamoDbTtlIdentity.toEventRecord(record.awsRecord());
node.put("eventSourceARN", pipe.getSource());
recordNodes.add(node);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java
index fc6c02fb37..7e35745b41 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java
@@ -13,7 +13,10 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
import java.util.Base64;
+import java.util.List;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
@@ -32,6 +35,9 @@ class DynamoDbKinesisStreamingIntegrationTest {
@Inject
KinesisStreamingForwarder forwarder;
+ @Inject
+ DynamoDbService dynamoDbService;
+
@BeforeAll
static void configureRestAssured() {
RestAssuredJsonUtils.configureAwsContentTypes();
@@ -293,7 +299,7 @@ class DynamoDbKinesisStreamingIntegrationTest {
@Test
@Order(12)
- void deleteItemForwardsRemoveEvent() {
+ void deleteItemForwardsRemoveEvent() throws Exception {
given()
.header("X-Amz-Target", "DynamoDB_20120810.DeleteItem")
.contentType(DYNAMODB_CONTENT_TYPE)
@@ -334,6 +340,8 @@ class DynamoDbKinesisStreamingIntegrationTest {
String lastEncoded = recordsResponse.jsonPath().getString("Records[" + (recordCount - 1) + "].Data");
String lastDecoded = new String(Base64.getDecoder().decode(lastEncoded));
assertTrue(lastDecoded.contains("\"eventName\":\"REMOVE\""), "Expected REMOVE event, got: " + lastDecoded);
+ assertTrue(new ObjectMapper().readTree(lastDecoded).get("userIdentity").isNull(),
+ "a DeleteItem removal is not made by the DynamoDB service: " + lastDecoded);
}
@Test
@@ -481,6 +489,77 @@ class DynamoDbKinesisStreamingIntegrationTest {
.body("DestinationStatus", equalTo("ACTIVE"));
}
+ @Test
+ @Order(31)
+ void ttlDeletionForwardsTheDynamoDbServiceIdentity() throws Exception {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTimeToLive")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "expireAt"}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ long past = Instant.now().getEpochSecond() - 3600;
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"Item\": {\"pk\": {\"S\": \"expired\"}, "
+ + "\"expireAt\": {\"N\": \"" + past + "\"}}}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ dynamoDbService.deleteExpiredItems();
+ awaitCdcDelivery();
+
+ JsonNode removal = null;
+ for (JsonNode payload : kinesisPayloads()) {
+ if ("REMOVE".equals(payload.path("eventName").asText())
+ && "expired".equals(payload.path("dynamodb").path("Keys").path("pk").path("S").asText())) {
+ removal = payload;
+ }
+ }
+ assertNotNull(removal, "the TTL deletion must be forwarded to Kinesis");
+ assertEquals(new ObjectMapper().readTree("{\"type\":\"Service\",\"principalId\":\"dynamodb.amazonaws.com\"}"),
+ removal.get("userIdentity"));
+ }
+
+ private List<JsonNode> kinesisPayloads() throws Exception {
+ String shardIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+ List<String> encoded = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + shardIterator + "\", \"Limit\": 100}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getList("Records.Data", String.class);
+ ObjectMapper mapper = new ObjectMapper();
+ List<JsonNode> payloads = new ArrayList<>();
+ for (String data : encoded) {
+ payloads.add(mapper.readTree(Base64.getDecoder().decode(data)));
+ }
+ return payloads;
+ }
+
@Test
@Order(40)
void enableLeavesDynamoDbStreamsDisabled() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
index 73ce06ab5a..2bf8f0799b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
@@ -398,4 +398,35 @@ class DynamoDbStreamServiceTest {
assertEquals(1, service.listStreams("ViewTypeTable", "000000000000", "us-east-1").size());
}
+ @Test
+ void captureEvent_ttlRemoval_isMarkedAsATtlDeletion() throws Exception {
+ TableDefinition table = sharedNameTable(ACCOUNT_A);
+ StreamDescription stream = service.enableStream(SHARED_TABLE, table.getTableArn(), "OLD_IMAGE", "us-east-1");
+
+ service.captureEvent("REMOVE", mapper.readTree("{\"userId\":{\"S\":\"a\"}}"), null, table, "us-east-1", true);
+
+ DynamoDbStreamRecord record = onlyRecordIn(stream.getStreamArn());
+ assertEquals("REMOVE", record.getEventName());
+ assertTrue(record.isTtlDeletion());
+ }
+
+ @Test
+ void captureEvent_deleteItemRemoval_isNotATtlDeletion() throws Exception {
+ TableDefinition table = sharedNameTable(ACCOUNT_A);
+ StreamDescription stream = service.enableStream(SHARED_TABLE, table.getTableArn(), "OLD_IMAGE", "us-east-1");
+
+ service.captureEvent("REMOVE", mapper.readTree("{\"userId\":{\"S\":\"a\"}}"), null, table, "us-east-1");
+
+ DynamoDbStreamRecord record = onlyRecordIn(stream.getStreamArn());
+ assertEquals("REMOVE", record.getEventName());
+ assertFalse(record.isTtlDeletion());
+ }
+
+ private DynamoDbStreamRecord onlyRecordIn(String streamArn) {
+ String iterator = service.getShardIterator(streamArn, DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);
+ List<DynamoDbStreamRecord> records = service.getRecords(iterator, 10).records();
+ assertEquals(1, records.size());
+ return records.get(0);
+ }
+
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
index 7de9fc8cfb..500b125f4c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
@@ -61,6 +61,11 @@ class DynamoDbTtlForwardingTest {
};
private static final String REGION = "us-east-1";
+ private static final String KINESIS_TTL_IDENTITY =
+ "{\"type\":\"Service\",\"principalId\":\"dynamodb.amazonaws.com\"}";
+ /** The DynamoDB Streams API capitalises the Identity members, as real AWS returns them. */
+ private static final String STREAMS_TTL_IDENTITY =
+ "{\"Type\":\"Service\",\"PrincipalId\":\"dynamodb.amazonaws.com\"}";
private final ObjectMapper mapper = new ObjectMapper();
@@ -141,6 +146,8 @@ class DynamoDbTtlForwardingTest {
for (KinesisRecord record : drained) {
JsonNode payload = mapper.readTree(record.getData());
assertEquals("REMOVE", payload.get("eventName").asText(), "TTL deletions forward as REMOVE");
+ assertEquals(mapper.readTree(KINESIS_TTL_IDENTITY), payload.get("userIdentity"),
+ "TTL deletions forward as made by the DynamoDB service");
}
assertEquals(0L, forwarder.getForwardFailureCount(), "happy path drops nothing");
@@ -218,7 +225,7 @@ class DynamoDbTtlForwardingTest {
doAnswer(invocation -> {
svc.deleteTable("TtlTable", REGION);
return null;
- }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+ }).when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));
svc.deleteExpiredItems();
@@ -277,11 +284,11 @@ class DynamoDbTtlForwardingTest {
doAnswer(invocation -> {
svc.deleteTable("TtlTable", REGION);
return null;
- }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+ }).doNothing().when(streamService).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));
svc.deleteScannedItems(scans);
- verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any());
+ verify(streamService, times(1)).captureEvent(eq("REMOVE"), any(), any(), any(), any(), eq(true));
}
@Test
@@ -393,4 +400,58 @@ class DynamoDbTtlForwardingTest {
assertEquals(1, kinesisRemoves.size(), "a skipped delete must not forward a Kinesis REMOVE record");
assertEquals("row-1", kinesisRemoves.get(0).get("dynamodb").get("Keys").get("pk").get("S").asText());
}
+
+ @Test
+ void getRecords_ttlSweepRemoval_carriesTheDynamoDbServiceIdentity() throws Exception {
+ StreamedTtlTable streamed = streamedTtlTable();
+ streamed.service().deleteExpiredItems();
+
+ JsonNode record = onlyRecord(streamed);
+ assertEquals("REMOVE", record.path("eventName").asText());
+ assertEquals("row-0", record.path("dynamodb").path("Keys").path("pk").path("S").asText());
+ assertEquals(mapper.readTree(STREAMS_TTL_IDENTITY), record.path("userIdentity"));
+ }
+
+ @Test
+ void getRecords_deleteItemRemoval_carriesNoIdentity() throws Exception {
+ StreamedTtlTable streamed = streamedTtlTable();
+ ObjectNode key = mapper.createObjectNode();
+ key.set("pk", stringAttr("row-0"));
+ streamed.service().deleteItem("TtlTable", key, REGION);
+
+ JsonNode record = onlyRecord(streamed);
+ assertEquals("REMOVE", record.path("eventName").asText());
+ assertFalse(record.has("userIdentity"), "a DeleteItem removal is not made by the DynamoDB service");
+ }
+
+ private record StreamedTtlTable(DynamoDbService service, NativeDynamoDbStreamsJsonHandler handler,
+ String streamArn) {}
+
+ /** A TTL table holding one expired item, whose stream starts after the item was written. */
+ private StreamedTtlTable streamedTtlTable() {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbStreamService streamService = new DynamoDbStreamService(mapper, tableStore);
+ RegionResolver regionResolver = new RegionResolver(REGION, "000000000000");
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, new InMemoryStorage<>(), regionResolver, streamService, null);
+ seedTtlTable(svc, 1);
+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();
+ StreamDescription sd = streamService.enableStream(
+ table.getTableName(), table.getTableArn(), "NEW_AND_OLD_IMAGES", REGION);
+ return new StreamedTtlTable(svc,
+ new NativeDynamoDbStreamsJsonHandler(streamService, svc, regionResolver, mapper), sd.getStreamArn());
+ }
+
+ private JsonNode onlyRecord(StreamedTtlTable streamed) {
+ ObjectNode iteratorRequest = mapper.createObjectNode()
+ .put("StreamArn", streamed.streamArn())
+ .put("ShardId", DynamoDbStreamService.SHARD_ID)
+ .put("ShardIteratorType", "TRIM_HORIZON");
+ JsonNode iterator = (JsonNode) streamed.handler().handle("GetShardIterator", iteratorRequest, REGION).getEntity();
+ ObjectNode recordsRequest = mapper.createObjectNode().put("ShardIterator", iterator.path("ShardIterator").asText());
+ JsonNode records = ((JsonNode) streamed.handler().handle("GetRecords", recordsRequest, REGION).getEntity())
+ .path("Records");
+ assertEquals(1, records.size(), "one stream record: " + records);
+ return records.get(0);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java
index c85c2ac86a..25835b1f08 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendIntegrationTest.java
@@ -20,6 +20,8 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import java.time.Duration;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -27,6 +29,7 @@ import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import static io.restassured.RestAssured.given;
+import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.endsWith;
@@ -203,6 +206,38 @@ class DynamoDbLocalBackendIntegrationTest {
assertEquals(CheckpointLifetime.STREAM, streamReader.checkpointLifetime());
}
+ @Test
+ void aTtlDeletionReadsAsRemoveByTheDynamoDbService() {
+ String table = tableName("ttl");
+ String streamArn = createTable(A_WEST, table, STREAMED)
+ .statusCode(200)
+ .extract().path("TableDescription.LatestStreamArn");
+ dynamoDb(A_WEST, "UpdateTimeToLive", "{\"TableName\": \"" + table
+ + "\", \"TimeToLiveSpecification\": {\"Enabled\": true, \"AttributeName\": \"expireAt\"}}")
+ .statusCode(200);
+ long past = Instant.now().getEpochSecond() - 3600;
+ dynamoDb(A_WEST, "PutItem", "{\"TableName\": \"" + table + "\", \"Item\": {\"pk\": {\"S\": \"expired\"}, "
+ + "\"expireAt\": {\"N\": \"" + past + "\"}}}")
+ .statusCode(200);
+ String shardId = streams(A_WEST, "DescribeStream", "{\"StreamArn\": \"" + streamArn + "\"}")
+ .statusCode(200)
+ .extract().path("StreamDescription.Shards[0].ShardId");
+
+ // DynamoDB Local deleted an expired item within about 15 seconds when measured.
+ await().atMost(Duration.ofSeconds(60)).pollInterval(Duration.ofSeconds(1)).untilAsserted(() -> {
+ String iterator = streams(A_WEST, "GetShardIterator", "{\"StreamArn\": \"" + streamArn
+ + "\", \"ShardId\": \"" + shardId + "\", \"ShardIteratorType\": \"TRIM_HORIZON\"}")
+ .statusCode(200)
+ .extract().path("ShardIterator");
+ streams(A_WEST, "GetRecords", "{\"ShardIterator\": \"" + iterator + "\"}")
+ .statusCode(200)
+ .body("Records.eventName", contains("INSERT", "REMOVE"))
+ .body("Records[1].dynamodb.Keys.pk.S", equalTo("expired"))
+ .body("Records[1].userIdentity.Type", equalTo("Service"))
+ .body("Records[1].userIdentity.PrincipalId", equalTo("dynamodb.amazonaws.com"));
+ });
+ }
+
@Test
void theDirectDeleteRemovesAProtectedTable() {
String table = tableName("protected");
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java
index 9c2f2dc57d..3d64822d74 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/backend/DynamoDbLocalBackendTest.java
@@ -278,6 +278,32 @@ class DynamoDbLocalBackendTest {
assertEquals(REGION, reply.body().path("Records").path(0).path("awsRegion").asText());
}
+ @Test
+ void timeToLiveDeletionReadsAsRemoveByTheDynamoDbService() throws Exception {
+ answer(Api.DYNAMODB_STREAMS, "GetRecords", 200, "{\"Records\":[{\"eventName\":\"UNKNOWN_TO_SDK_VERSION\","
+ + "\"awsRegion\":\"ddblocal\",\"dynamodb\":{\"Keys\":{\"pk\":{\"S\":\"a\"}},"
+ + "\"OldImage\":{\"pk\":{\"S\":\"a\"}}}}]}");
+
+ Reply reply = execute(Api.DYNAMODB_STREAMS, "GetRecords", "{\"ShardIterator\":\"it\"}");
+
+ JsonNode record = reply.body().path("Records").path(0);
+ assertEquals("REMOVE", record.path("eventName").asText());
+ assertEquals(json("{\"Type\":\"Service\",\"PrincipalId\":\"dynamodb.amazonaws.com\"}"),
+ record.path("userIdentity"));
+ assertEquals("a", record.path("dynamodb").path("OldImage").path("pk").path("S").asText());
+ }
+
+ @Test
+ void writeRecordsKeepTheirEventNameAndCarryNoIdentity() throws Exception {
+ String records = "[{\"eventName\":\"INSERT\",\"dynamodb\":{}},{\"eventName\":\"MODIFY\",\"dynamodb\":{}},"
+ + "{\"eventName\":\"REMOVE\",\"dynamodb\":{}}]";
+ answer(Api.DYNAMODB_STREAMS, "GetRecords", 200, "{\"Records\":" + records + "}");
+
+ Reply reply = execute(Api.DYNAMODB_STREAMS, "GetRecords", "{\"ShardIterator\":\"it\"}");
+
+ assertEquals(json(records), reply.body().path("Records"));
+ }
+
@Test
void arnLookingItemAttributeUntouched() throws Exception {
String item = "{\"TableArn\":{\"S\":\"" + LOCAL + "table/x\"},"
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
index 1e3e4f54f6..a37622624b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java
@@ -543,6 +543,45 @@ class DynamoDbStreamsEventSourcePollerTest {
assertEquals("s2", esm.getShardSequenceNumbers().get(SHARD_ID));
}
+ @Test
+ void userIdentityFilterDeliversOnlyTheTtlDeletion() {
+ stubTrimHorizon(List.of(
+ ddbRecord("s1", "INSERT", "{\"status\":{\"S\":\"active\"}}"),
+ removal("s2", false),
+ removal("s3", true)));
+ when(executorService.invoke(any(), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(new InvokeResult());
+ EventSourceMapping esm = filterEsm(
+ "{\"userIdentity\":{\"type\":[\"Service\"],\"principalId\":[\"dynamodb.amazonaws.com\"]}}");
+
+ pollerWith(mock(EsmStore.class)).pollAndInvoke(esm);
+
+ ArgumentCaptor<byte[]> payload = ArgumentCaptor.forClass(byte[].class);
+ verify(executorService, timeout(2000)).invoke(any(), payload.capture(), eq(InvocationType.RequestResponse));
+ JsonNode records = readRecords(payload.getValue());
+ assertEquals(1, records.size());
+ assertEquals("s3", records.get(0).path("dynamodb").path("SequenceNumber").asText());
+ assertEquals("REMOVE", records.get(0).path("eventName").asText());
+ assertEquals(OBJECT_MAPPER.createObjectNode().put("type", "Service").put("principalId", "dynamodb.amazonaws.com"),
+ records.get(0).path("userIdentity"));
+ }
+
+ /**
+ * A REMOVE record as GetRecords returns it. One that time to live made carries the DynamoDB service
+ * identity with the Streams API's capitalised member names, as AWS writes it.
+ */
+ private DynamoDbStreamReader.Record removal(String seq, boolean byTtl) {
+ DynamoDbStreamReader.Record record = ddbRecord(seq, "REMOVE", "{\"status\":{\"S\":\"expired\"}}");
+ ObjectNode dynamodb = (ObjectNode) record.awsRecord().path("dynamodb");
+ dynamodb.set("OldImage", dynamodb.remove("NewImage"));
+ if (byTtl) {
+ ((ObjectNode) record.awsRecord()).putObject("userIdentity")
+ .put("PrincipalId", "dynamodb.amazonaws.com")
+ .put("Type", "Service");
+ }
+ return record;
+ }
+
@Test
void partialBatchFailureCheckpointsBeforeLowestFailedRecord() {
stubTrimHorizon(List.of(
diff --git a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
index beff9ac091..9c9ff83c69 100644
--- a/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/pipes/PipesPollerTest.java
@@ -549,6 +549,29 @@ class PipesPollerTest {
assertEquals(original, active.awsRecord(), "the reader's record is never mutated");
}
+ @Test
+ void userIdentityFilterDeliversOnlyTheTtlDeletionWithTheEventRecordIdentity() throws Exception {
+ Pipe pipe = dynamoDbPipe("{}");
+ pipe.setSourceParameters(MAPPER.readTree("""
+ {"DynamoDBStreamParameters":{"StartingPosition":"TRIM_HORIZON"},
+ "FilterCriteria":{"Filters":[{"Pattern":
+ "{\\"userIdentity\\":{\\"type\\":[\\"Service\\"],\\"principalId\\":[\\"dynamodb.amazonaws.com\\"]}}"}]}}"""));
+ streams.shard.add(ddbRemoval("001", false));
+ DynamoDbStreamReader.Record ttlRemoval = ddbRemoval("002", true);
+ JsonNode original = ttlRemoval.awsRecord().deepCopy();
+ streams.shard.add(ttlRemoval);
+
+ poller.startPolling(pipe);
+ poller.pollDynamoDbStreams(pipe, "us-east-1");
+
+ assertEquals(List.of(List.of("002")), deliveredSequences(pipe));
+ ArgumentCaptor<String> payload = ArgumentCaptor.forClass(String.class);
+ verify(targetInvoker).invoke(eq(pipe), payload.capture(), eq("us-east-1"));
+ assertEquals(MAPPER.createObjectNode().put("type", "Service").put("principalId", "dynamodb.amazonaws.com"),
+ MAPPER.readTree(payload.getValue()).path("Records").path(0).path("userIdentity"));
+ assertEquals(original, ttlRemoval.awsRecord(), "the reader's record is never mutated");
+ }
+
@Test
void trimmedDynamoDbCheckpointRestartsAtTheTrimHorizon() throws Exception {
Pipe pipe = dynamoDbPipe("{}");
@@ -641,6 +664,22 @@ class PipesPollerTest {
return new DynamoDbStreamReader.Record(sequenceNumber, awsRecord);
}
+ /**
+ * A REMOVE record as GetRecords returns it. One that time to live made carries the DynamoDB service
+ * identity with the Streams API's capitalised member names, as AWS writes it.
+ */
+ private static DynamoDbStreamReader.Record ddbRemoval(String sequenceNumber, boolean byTtl) {
+ DynamoDbStreamReader.Record record = ddbRecord(sequenceNumber, "expired");
+ ObjectNode awsRecord = (ObjectNode) record.awsRecord();
+ awsRecord.put("eventName", "REMOVE");
+ ObjectNode dynamodb = (ObjectNode) awsRecord.path("dynamodb");
+ dynamodb.set("OldImage", dynamodb.remove("NewImage"));
+ if (byTtl) {
+ awsRecord.putObject("userIdentity").put("PrincipalId", "dynamodb.amazonaws.com").put("Type", "Service");
+ }
+ return record;
+ }
+
/** The sequence numbers of each batch the Lambda target was invoked with, in invocation order. */
private List<List<String>> deliveredSequences(Pipe pipe) throws Exception {
ArgumentCaptor<String> payloads = ArgumentCaptor.forClass(String.class);