Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): add EnableKinesisStreamingDestination support (#427)

yusuke-uranaka467f5e0f79ce
diff --git a/compatibility-tests/sdk-test-awscli/test/dynamodb.bats b/compatibility-tests/sdk-test-awscli/test/dynamodb.bats
index 1d1b822af5..d7efd71f73 100644
--- a/compatibility-tests/sdk-test-awscli/test/dynamodb.bats
+++ b/compatibility-tests/sdk-test-awscli/test/dynamodb.bats
@@ -575,6 +575,95 @@ teardown() {
[ "$count" = "1" ]
}
+# --- DynamoDB Kinesis Streaming Destination Tests ---
+
+@test "DynamoDB: enable kinesis streaming destination" {
+ STREAM_NAME="bats-kinesis-$(unique_name)"
+
+ aws_cmd kinesis create-stream --stream-name "$STREAM_NAME" --shard-count 1 >/dev/null
+
+ aws_cmd dynamodb create-table \
+ --table-name "$TABLE_NAME" \
+ --attribute-definitions AttributeName=pk,AttributeType=S \
+ --key-schema AttributeName=pk,KeyType=HASH \
+ --billing-mode PAY_PER_REQUEST \
+ --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES >/dev/null
+
+ ddb_wait_table "$TABLE_NAME"
+
+ STREAM_ARN=$(aws_cmd kinesis describe-stream-summary --stream-name "$STREAM_NAME" | jq -r '.StreamDescriptionSummary.StreamARN')
+
+ run aws_cmd dynamodb enable-kinesis-streaming-destination \
+ --table-name "$TABLE_NAME" \
+ --stream-arn "$STREAM_ARN"
+ assert_success
+ status=$(json_get "$output" '.DestinationStatus')
+ [ "$status" = "ACTIVE" ]
+
+ aws_cmd kinesis delete-stream --stream-name "$STREAM_NAME" >/dev/null 2>&1 || true
+}
+
+@test "DynamoDB: describe kinesis streaming destination" {
+ STREAM_NAME="bats-kinesis-$(unique_name)"
+
+ aws_cmd kinesis create-stream --stream-name "$STREAM_NAME" --shard-count 1 >/dev/null
+
+ aws_cmd dynamodb create-table \
+ --table-name "$TABLE_NAME" \
+ --attribute-definitions AttributeName=pk,AttributeType=S \
+ --key-schema AttributeName=pk,KeyType=HASH \
+ --billing-mode PAY_PER_REQUEST \
+ --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES >/dev/null
+
+ ddb_wait_table "$TABLE_NAME"
+
+ STREAM_ARN=$(aws_cmd kinesis describe-stream-summary --stream-name "$STREAM_NAME" | jq -r '.StreamDescriptionSummary.StreamARN')
+
+ aws_cmd dynamodb enable-kinesis-streaming-destination \
+ --table-name "$TABLE_NAME" \
+ --stream-arn "$STREAM_ARN" >/dev/null
+
+ run aws_cmd dynamodb describe-kinesis-streaming-destination \
+ --table-name "$TABLE_NAME"
+ assert_success
+ count=$(echo "$output" | jq '.KinesisDataStreamDestinations | length')
+ [ "$count" = "1" ]
+ dest_status=$(echo "$output" | jq -r '.KinesisDataStreamDestinations[0].DestinationStatus')
+ [ "$dest_status" = "ACTIVE" ]
+
+ aws_cmd kinesis delete-stream --stream-name "$STREAM_NAME" >/dev/null 2>&1 || true
+}
+
+@test "DynamoDB: disable kinesis streaming destination" {
+ STREAM_NAME="bats-kinesis-$(unique_name)"
+
+ aws_cmd kinesis create-stream --stream-name "$STREAM_NAME" --shard-count 1 >/dev/null
+
+ aws_cmd dynamodb create-table \
+ --table-name "$TABLE_NAME" \
+ --attribute-definitions AttributeName=pk,AttributeType=S \
+ --key-schema AttributeName=pk,KeyType=HASH \
+ --billing-mode PAY_PER_REQUEST \
+ --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES >/dev/null
+
+ ddb_wait_table "$TABLE_NAME"
+
+ STREAM_ARN=$(aws_cmd kinesis describe-stream-summary --stream-name "$STREAM_NAME" | jq -r '.StreamDescriptionSummary.StreamARN')
+
+ aws_cmd dynamodb enable-kinesis-streaming-destination \
+ --table-name "$TABLE_NAME" \
+ --stream-arn "$STREAM_ARN" >/dev/null
+
+ run aws_cmd dynamodb disable-kinesis-streaming-destination \
+ --table-name "$TABLE_NAME" \
+ --stream-arn "$STREAM_ARN"
+ assert_success
+ status=$(json_get "$output" '.DestinationStatus')
+ [ "$status" = "DISABLED" ]
+
+ aws_cmd kinesis delete-stream --stream-name "$STREAM_NAME" >/dev/null 2>&1 || true
+}
+
@test "DynamoDB: scan with contains on String Set" {
aws_cmd dynamodb create-table \
--table-name "$TABLE_NAME" \
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index c2b4d550c9..30e5bb1d47 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -1,12 +1,14 @@
package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.core.common.AwsErrorResponse;
+import io.github.hectorvent.floci.core.common.AwsException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsJsonController;
import io.github.hectorvent.floci.services.dynamodb.model.*;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
@@ -22,13 +24,15 @@ public class DynamoDbJsonHandler {
private final DynamoDbService dynamoDbService;
private final DynamoDbStreamService dynamoDbStreamService;
+ private final KinesisService kinesisService;
private final ObjectMapper objectMapper;
@Inject
public DynamoDbJsonHandler(DynamoDbService dynamoDbService, DynamoDbStreamService dynamoDbStreamService,
- ObjectMapper objectMapper) {
+ KinesisService kinesisService, ObjectMapper objectMapper) {
this.dynamoDbService = dynamoDbService;
this.dynamoDbStreamService = dynamoDbStreamService;
+ this.kinesisService = kinesisService;
this.objectMapper = objectMapper;
}
@@ -54,6 +58,9 @@ public class DynamoDbJsonHandler {
case "TagResource" -> handleTagResource(request, region);
case "UntagResource" -> handleUntagResource(request, region);
case "ListTagsOfResource" -> handleListTagsOfResource(request, region);
+ case "EnableKinesisStreamingDestination" -> handleEnableKinesisStreamingDestination(request, region);
+ case "DisableKinesisStreamingDestination" -> handleDisableKinesisStreamingDestination(request, region);
+ case "DescribeKinesisStreamingDestination" -> handleDescribeKinesisStreamingDestination(request, region);
default -> Response.status(400)
.entity(new AwsErrorResponse("UnknownOperationException", "Operation " + action + " is not supported."))
.build();
@@ -610,6 +617,101 @@ public class DynamoDbJsonHandler {
return Response.ok(response).build();
}
+ private Response handleEnableKinesisStreamingDestination(JsonNode request, String region) {
+ String tableName = request.path("TableName").asText();
+ String streamArn = request.path("StreamArn").asText();
+
+ TableDefinition table = dynamoDbService.describeTable(tableName, region);
+
+ String streamName = streamArn.substring(streamArn.lastIndexOf('/') + 1);
+ try {
+ kinesisService.describeStream(streamName, region);
+ } catch (AwsException e) {
+ throw new AwsException("ResourceNotFoundException",
+ "Kinesis stream not found: " + streamArn, 400);
+ }
+
+ Optional<KinesisStreamingDestination> existing = table.findKinesisStreamingDestination(streamArn);
+ if (existing.isPresent() && "ACTIVE".equals(existing.get().getDestinationStatus())) {
+ throw new AwsException("ValidationException",
+ "Table already has an active Kinesis streaming destination with this stream ARN", 400);
+ }
+
+ if (existing.isPresent()) {
+ existing.get().setDestinationStatus("ACTIVE");
+ existing.get().setDestinationStatusDescription("Kinesis streaming is enabled for this table");
+ } else {
+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(streamArn));
+ }
+
+ if (!table.isStreamEnabled()) {
+ StreamDescription sd = dynamoDbStreamService.enableStream(
+ tableName, table.getTableArn(), "NEW_AND_OLD_IMAGES", region);
+ table.setStreamEnabled(true);
+ table.setStreamArn(sd.getStreamArn());
+ table.setStreamViewType("NEW_AND_OLD_IMAGES");
+ }
+
+ dynamoDbService.persistTable(tableName, table, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("TableName", tableName);
+ response.put("StreamArn", streamArn);
+ response.put("DestinationStatus", "ACTIVE");
+ response.put("DestinationStatusDescription", "Kinesis streaming is enabled for this table");
+ return Response.ok(response).build();
+ }
+
+ private Response handleDisableKinesisStreamingDestination(JsonNode request, String region) {
+ String tableName = request.path("TableName").asText();
+ String streamArn = request.path("StreamArn").asText();
+
+ TableDefinition table = dynamoDbService.describeTable(tableName, region);
+
+ Optional<KinesisStreamingDestination> existing = table.findKinesisStreamingDestination(streamArn);
+ if (existing.isEmpty()) {
+ throw new AwsException("ResourceNotFoundException",
+ "Kinesis streaming destination not found for stream: " + streamArn, 400);
+ }
+
+ if ("DISABLED".equals(existing.get().getDestinationStatus())) {
+ throw new AwsException("ValidationException",
+ "Kinesis streaming destination is already disabled for stream: " + streamArn, 400);
+ }
+
+ existing.get().setDestinationStatus("DISABLED");
+ existing.get().setDestinationStatusDescription("Kinesis streaming is disabled for this table");
+ dynamoDbService.persistTable(tableName, table, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("TableName", tableName);
+ response.put("StreamArn", streamArn);
+ response.put("DestinationStatus", "DISABLED");
+ response.put("DestinationStatusDescription", "Kinesis streaming is disabled for this table");
+ return Response.ok(response).build();
+ }
+
+ private Response handleDescribeKinesisStreamingDestination(JsonNode request, String region) {
+ String tableName = request.path("TableName").asText();
+ TableDefinition table = dynamoDbService.describeTable(tableName, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.put("TableName", tableName);
+
+ ArrayNode destinations = objectMapper.createArrayNode();
+ for (KinesisStreamingDestination dest : table.getKinesisStreamingDestinations()) {
+ ObjectNode destNode = objectMapper.createObjectNode();
+ destNode.put("StreamArn", dest.getStreamArn());
+ destNode.put("DestinationStatus", dest.getDestinationStatus());
+ destNode.put("DestinationStatusDescription", dest.getDestinationStatusDescription());
+ destNode.put("ApproximateCreationDateTimePrecision",
+ dest.getApproximateCreationDateTimePrecision());
+ destinations.add(destNode);
+ }
+ response.set("KinesisDataStreamDestinations", destinations);
+ return Response.ok(response).build();
+ }
+
private ObjectNode tableToNode(TableDefinition table) {
ObjectNode node = objectMapper.createObjectNode();
node.put("TableName", table.getTableName());
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index a0c49cafb2..0c9f37077e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -181,6 +181,10 @@ public class DynamoDbService {
return table;
}
+ public void persistTable(String tableName, TableDefinition table, String region) {
+ tableStore.put(regionKey(region, tableName), table);
+ }
+
public void deleteTable(String tableName) {
deleteTable(tableName, regionResolver.getDefaultRegion());
}
@@ -241,9 +245,10 @@ public class DynamoDbService {
persistItems(storageKey);
LOG.debugv("Put item in {0}: key={1}", tableName, itemKey);
+ String eventName = existing == null ? "INSERT" : "MODIFY";
if (streamService != null) {
- streamService.captureEvent(tableName,
- existing == null ? "INSERT" : "MODIFY", existing, item, table, region);
+ streamService.captureEvent(tableName, eventName, existing, item, table, region);
+ streamService.forwardToKinesisDestinations(eventName, existing, item, table, region);
}
}
@@ -295,6 +300,7 @@ public class DynamoDbService {
if (streamService != null && removed != null) {
streamService.captureEvent(tableName, "REMOVE", removed, null, table, region);
+ streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);
}
return removed;
@@ -371,6 +377,7 @@ public class DynamoDbService {
if (streamService != null) {
streamService.captureEvent(tableName, "MODIFY", existing, item, table, region);
+ streamService.forwardToKinesisDestinations("MODIFY", existing, item, table, region);
}
return new UpdateResult(item, existing);
@@ -813,6 +820,7 @@ public class DynamoDbService {
JsonNode removed = items.remove(itemKey);
if (removed != null && streamService != null) {
streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);
+ streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);
}
}
persistItems(storageKey);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index d5b3e9f3c4..11d6d2df0f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -5,8 +5,10 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -44,16 +46,24 @@ public class DynamoDbStreamService {
private final AtomicLong sequenceCounter = new AtomicLong(0);
private final ObjectMapper objectMapper;
+ private KinesisService kinesisService;
@Inject
- public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory) {
+ public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory,
+ KinesisService kinesisService) {
this(objectMapper, storageFactory.create("dynamodb", "dynamodb-tables.json",
- new TypeReference<Map<String, TableDefinition>>() {}));
+ new TypeReference<Map<String, TableDefinition>>() {}), kinesisService);
}
/** Package-private constructor for testing. */
DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore) {
+ this(objectMapper, tableStore, null);
+ }
+
+ DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore,
+ KinesisService kinesisService) {
this.objectMapper = objectMapper;
+ this.kinesisService = kinesisService;
loadPersistedStreams(tableStore);
}
@@ -165,6 +175,85 @@ public class DynamoDbStreamService {
}
}
+ public void forwardToKinesisDestinations(String eventName, JsonNode oldItem, JsonNode newItem,
+ TableDefinition table, String region) {
+ if (kinesisService == null) return;
+
+ List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();
+ if (destinations == null || destinations.isEmpty()) return;
+
+ Instant now = Instant.now();
+ JsonNode sourceItem = newItem != null ? newItem : oldItem;
+ ObjectNode keys = buildKeys(sourceItem, table);
+
+ for (KinesisStreamingDestination dest : destinations) {
+ if (!"ACTIVE".equals(dest.getDestinationStatus())) continue;
+
+ try {
+ ObjectNode payload = buildKinesisPayload(eventName, keys, newItem, oldItem,
+ table.getTableName(), region, now);
+ byte[] data = objectMapper.writeValueAsBytes(payload);
+
+ String partitionKey = extractPartitionKey(keys, table);
+ String streamName = extractStreamName(dest.getStreamArn());
+
+ kinesisService.putRecord(streamName, data, partitionKey, region);
+ LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",
+ streamName, eventName, table.getTableName());
+ } catch (Exception e) {
+ LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",
+ dest.getStreamArn(), e.getMessage());
+ }
+ }
+ }
+
+ private ObjectNode buildKinesisPayload(String eventName, JsonNode keys,
+ JsonNode newImage, JsonNode oldImage,
+ String tableName, String region, Instant timestamp) {
+ ObjectNode payload = objectMapper.createObjectNode();
+ payload.put("awsRegion", region);
+ payload.put("eventID", UUID.randomUUID().toString());
+ payload.put("eventName", eventName);
+ payload.putNull("userIdentity");
+ payload.put("recordFormat", "application/json");
+ payload.put("tableName", tableName);
+ payload.put("eventSource", "aws:dynamodb");
+
+ ObjectNode dynamodb = objectMapper.createObjectNode();
+ dynamodb.put("ApproximateCreationDateTime", timestamp.toEpochMilli());
+ if (keys != null) {
+ dynamodb.set("Keys", keys);
+ }
+ if (newImage != null) {
+ dynamodb.set("NewImage", newImage);
+ }
+ if (oldImage != null) {
+ dynamodb.set("OldImage", oldImage);
+ }
+ dynamodb.put("SizeBytes", 0);
+ dynamodb.put("ApproximateCreationDateTimePrecision", "MILLISECOND");
+ payload.set("dynamodb", dynamodb);
+
+ return payload;
+ }
+
+ private String extractPartitionKey(JsonNode keys, TableDefinition table) {
+ if (keys == null || keys.isEmpty()) return "default";
+ String pkName = table.getPartitionKeyName();
+ JsonNode pkValue = keys.get(pkName);
+ if (pkValue == null) return "default";
+ if (pkValue.has("S")) return pkValue.get("S").asText();
+ if (pkValue.has("N")) return pkValue.get("N").asText();
+ if (pkValue.has("B")) return pkValue.get("B").asText();
+ return pkValue.toString();
+ }
+
+ private String extractStreamName(String streamArn) {
+ int idx = streamArn.lastIndexOf('/');
+ if (idx >= 0) return streamArn.substring(idx + 1);
+ return streamArn;
+ }
+
private ObjectNode buildKeys(JsonNode item, TableDefinition table) {
ObjectNode keys = objectMapper.createObjectNode();
if (item == null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/KinesisStreamingDestination.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/KinesisStreamingDestination.java
new file mode 100644
index 0000000000..15837b7422
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/KinesisStreamingDestination.java
@@ -0,0 +1,37 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class KinesisStreamingDestination {
+
+ private String streamArn;
+ private String destinationStatus;
+ private String destinationStatusDescription;
+ private String approximateCreationDateTimePrecision;
+
+ public KinesisStreamingDestination() {}
+
+ public KinesisStreamingDestination(String streamArn) {
+ this.streamArn = streamArn;
+ this.destinationStatus = "ACTIVE";
+ this.destinationStatusDescription = "Kinesis streaming is enabled for this table";
+ this.approximateCreationDateTimePrecision = "MILLISECOND";
+ }
+
+ public String getStreamArn() { return streamArn; }
+ public void setStreamArn(String streamArn) { this.streamArn = streamArn; }
+
+ public String getDestinationStatus() { return destinationStatus; }
+ public void setDestinationStatus(String destinationStatus) { this.destinationStatus = destinationStatus; }
+
+ public String getDestinationStatusDescription() { return destinationStatusDescription; }
+ public void setDestinationStatusDescription(String desc) { this.destinationStatusDescription = desc; }
+
+ public String getApproximateCreationDateTimePrecision() { return approximateCreationDateTimePrecision; }
+ public void setApproximateCreationDateTimePrecision(String precision) {
+ this.approximateCreationDateTimePrecision = precision;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
index 6adeb0c63b..7b2e4c5b9f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableDefinition.java
@@ -35,6 +35,7 @@ public class TableDefinition {
private boolean streamEnabled;
private String streamArn;
private String streamViewType;
+ private List<KinesisStreamingDestination> kinesisStreamingDestinations;
public TableDefinition() {
this.keySchema = new ArrayList<>();
@@ -42,6 +43,7 @@ public class TableDefinition {
this.tags = new HashMap<>();
this.globalSecondaryIndexes = new ArrayList<>();
this.localSecondaryIndexes = new ArrayList<>();
+ this.kinesisStreamingDestinations = new ArrayList<>();
}
public TableDefinition(String tableName,
@@ -66,6 +68,7 @@ public class TableDefinition {
this.tags = new HashMap<>();
this.globalSecondaryIndexes = new ArrayList<>();
this.localSecondaryIndexes = new ArrayList<>();
+ this.kinesisStreamingDestinations = new ArrayList<>();
}
public String getTableName() { return tableName; }
@@ -126,6 +129,19 @@ public class TableDefinition {
public String getStreamViewType() { return streamViewType; }
public void setStreamViewType(String streamViewType) { this.streamViewType = streamViewType; }
+ public List<KinesisStreamingDestination> getKinesisStreamingDestinations() {
+ return kinesisStreamingDestinations != null ? kinesisStreamingDestinations : new ArrayList<>();
+ }
+ public void setKinesisStreamingDestinations(List<KinesisStreamingDestination> destinations) {
+ this.kinesisStreamingDestinations = destinations != null ? destinations : new ArrayList<>();
+ }
+
+ public Optional<KinesisStreamingDestination> findKinesisStreamingDestination(String streamArn) {
+ return getKinesisStreamingDestinations().stream()
+ .filter(d -> streamArn.equals(d.getStreamArn()))
+ .findFirst();
+ }
+
/** Returns the partition key attribute name. */
public String getPartitionKeyName() {
return keySchema.stream()
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java
new file mode 100644
index 0000000000..d394f758a5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbKinesisStreamingIntegrationTest.java
@@ -0,0 +1,502 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.Response;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.Base64;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbKinesisStreamingIntegrationTest {
+
+ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String KINESIS_CONTENT_TYPE = "application/x-amz-json-1.1";
+ private static String kinesisStreamArn;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void setupKinesisStream() {
+ given()
+ .header("X-Amz-Target", "Kinesis_20131202.CreateStream")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardCount": 1}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ kinesisStreamArn = given()
+ .header("X-Amz-Target", "Kinesis_20131202.DescribeStreamSummary")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("StreamDescriptionSummary.StreamARN");
+
+ assertNotNull(kinesisStreamArn);
+ }
+
+ @Test
+ @Order(2)
+ void setupDynamoDbTable() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("TableDescription.TableName", equalTo("StreamingTable"));
+ }
+
+ @Test
+ @Order(3)
+ void enableKinesisStreamingDestination() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("TableName", equalTo("StreamingTable"))
+ .body("StreamArn", equalTo(kinesisStreamArn))
+ .body("DestinationStatus", equalTo("ACTIVE"));
+ }
+
+ @Test
+ @Order(4)
+ void describeKinesisStreamingDestination() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "StreamingTable"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("TableName", equalTo("StreamingTable"))
+ .body("KinesisDataStreamDestinations.size()", equalTo(1))
+ .body("KinesisDataStreamDestinations[0].StreamArn", equalTo(kinesisStreamArn))
+ .body("KinesisDataStreamDestinations[0].DestinationStatus", equalTo("ACTIVE"))
+ .body("KinesisDataStreamDestinations[0].ApproximateCreationDateTimePrecision", equalTo("MILLISECOND"));
+ }
+
+ @Test
+ @Order(5)
+ void enableDuplicateDestinationFails() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
+ @Test
+ @Order(6)
+ void enableWithNonExistentStreamFails() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "StreamingTable", "StreamArn": "arn:aws:kinesis:us-east-1:000000000000:stream/no-such-stream"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+
+ @Test
+ @Order(7)
+ void enableWithNonExistentTableFails() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"NoSuchTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400);
+ }
+
+ @Test
+ @Order(10)
+ void putItemForwardsToKinesis() throws Exception {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "Item": {"pk": {"S": "k1"}, "data": {"S": "hello"}}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ String shardIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ Response recordsResponse = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + shardIterator + "\", \"Limit\": 10}")
+ .when()
+ .post("/");
+
+ recordsResponse.then().statusCode(200);
+
+ int recordCount = recordsResponse.jsonPath().getInt("Records.size()");
+ assertTrue(recordCount >= 1, "Expected at least 1 Kinesis record, got " + recordCount);
+
+ String encodedData = recordsResponse.jsonPath().getString("Records[0].Data");
+ String decoded = new String(Base64.getDecoder().decode(encodedData));
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode payload = mapper.readTree(decoded);
+ assertEquals("INSERT", payload.get("eventName").asText());
+ assertEquals("StreamingTable", payload.get("tableName").asText());
+ assertEquals("aws:dynamodb", payload.get("eventSource").asText());
+
+ JsonNode dynamodb = payload.get("dynamodb");
+ assertNotNull(dynamodb, "dynamodb node must be present");
+ assertNotNull(dynamodb.get("Keys"), "Keys must be present");
+ assertNotNull(dynamodb.get("NewImage"), "NewImage must be present");
+ assertNotNull(dynamodb.get("SizeBytes"), "SizeBytes must be present");
+ assertNotNull(dynamodb.get("ApproximateCreationDateTimePrecision"),
+ "ApproximateCreationDateTimePrecision must be present in dynamodb node");
+
+ long timestamp = dynamodb.get("ApproximateCreationDateTime").asLong();
+ long nowMillis = System.currentTimeMillis();
+ assertTrue(timestamp > nowMillis - 60_000 && timestamp <= nowMillis + 5_000,
+ "ApproximateCreationDateTime should be in milliseconds (recent), got: " + timestamp);
+
+ assertFalse(dynamodb.has("SequenceNumber"), "SequenceNumber should not be in Kinesis payload");
+ assertFalse(dynamodb.has("StreamViewType"), "StreamViewType should not be in Kinesis payload");
+ }
+
+ @Test
+ @Order(11)
+ void updateItemForwardsModifyEvent() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "Key": {"pk": {"S": "k1"}},
+ "UpdateExpression": "SET #d = :v",
+ "ExpressionAttributeNames": {"#d": "data"},
+ "ExpressionAttributeValues": {":v": {"S": "updated"}}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ String shardIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ Response recordsResponse = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + shardIterator + "\", \"Limit\": 10}")
+ .when()
+ .post("/");
+
+ int recordCount = recordsResponse.jsonPath().getInt("Records.size()");
+ assertTrue(recordCount >= 2, "Expected at least 2 records (INSERT + MODIFY), got " + recordCount);
+
+ String lastEncoded = recordsResponse.jsonPath().getString("Records[" + (recordCount - 1) + "].Data");
+ String lastDecoded = new String(Base64.getDecoder().decode(lastEncoded));
+ assertTrue(lastDecoded.contains("\"eventName\":\"MODIFY\""), "Expected MODIFY event, got: " + lastDecoded);
+ }
+
+ @Test
+ @Order(12)
+ void deleteItemForwardsRemoveEvent() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DeleteItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "Key": {"pk": {"S": "k1"}}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ String shardIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ Response recordsResponse = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + shardIterator + "\", \"Limit\": 10}")
+ .when()
+ .post("/");
+
+ int recordCount = recordsResponse.jsonPath().getInt("Records.size()");
+ assertTrue(recordCount >= 3, "Expected at least 3 records (INSERT + MODIFY + REMOVE), got " + recordCount);
+
+ String lastEncoded = recordsResponse.jsonPath().getString("Records[" + (recordCount - 1) + "].Data");
+ String lastDecoded = new String(Base64.getDecoder().decode(lastEncoded));
+ assertTrue(lastDecoded.contains("\"eventName\":\"REMOVE\""), "Expected REMOVE event, got: " + lastDecoded);
+ }
+
+ @Test
+ @Order(20)
+ void disableKinesisStreamingDestination() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DisableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DestinationStatus", equalTo("DISABLED"));
+ }
+
+ @Test
+ @Order(21)
+ void describeAfterDisable() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "StreamingTable"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("KinesisDataStreamDestinations[0].DestinationStatus", equalTo("DISABLED"));
+ }
+
+ @Test
+ @Order(22)
+ void putItemAfterDisableDoesNotForward() {
+ String beforeIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ int beforeCount = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + beforeIterator + "\", \"Limit\": 100}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getInt("Records.size()");
+
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "StreamingTable",
+ "Item": {"pk": {"S": "k-after-disable"}, "data": {"S": "should not appear"}}
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ String afterIterator = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetShardIterator")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("""
+ {"StreamName": "ddb-streaming-test", "ShardId": "shardId-000000000000", "ShardIteratorType": "TRIM_HORIZON"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("ShardIterator");
+
+ int afterCount = given()
+ .header("X-Amz-Target", "Kinesis_20131202.GetRecords")
+ .contentType(KINESIS_CONTENT_TYPE)
+ .body("{\"ShardIterator\": \"" + afterIterator + "\", \"Limit\": 100}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getInt("Records.size()");
+
+ assertEquals(beforeCount, afterCount,
+ "No new Kinesis records should be added after destination is disabled");
+ }
+
+ @Test
+ @Order(23)
+ void disableAlreadyDisabledFails() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DisableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
+ @Test
+ @Order(24)
+ void disableNonExistentDestinationFails() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DisableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "StreamingTable", "StreamArn": "arn:aws:kinesis:us-east-1:000000000000:stream/no-such"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ResourceNotFoundException"));
+ }
+
+ @Test
+ @Order(30)
+ void reEnableAfterDisable() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"StreamingTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DestinationStatus", equalTo("ACTIVE"));
+ }
+
+ @Test
+ @Order(40)
+ void enableAutoEnablesStreamsIfDisabled() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "NoStreamTable",
+ "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.EnableKinesisStreamingDestination")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"NoStreamTable\", \"StreamArn\": \"" + kinesisStreamArn + "\"}")
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DestinationStatus", equalTo("ACTIVE"));
+
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeTable")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {"TableName": "NoStreamTable"}
+ """)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("Table.StreamSpecification.StreamEnabled", equalTo(true));
+ }
+}