Light, fluffy, and always free - The AWS Local Emulator alternative
refactor(dynamodb): extract KinesisStreamingForwarder from DynamoDbStreamService (#428)
yusuke-uranakaa0cd0ac511b4
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 0c9f37077e..02a5730a3f 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -38,40 +38,44 @@ public class DynamoDbService {private final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();private final RegionResolver regionResolver;private DynamoDbStreamService streamService;+ private KinesisStreamingForwarder kinesisForwarder;@Injectpublic DynamoDbService(StorageFactory storageFactory, RegionResolver regionResolver,- DynamoDbStreamService streamService) {+ DynamoDbStreamService streamService,+ KinesisStreamingForwarder kinesisForwarder) {this(storageFactory.create("dynamodb", "dynamodb-tables.json",new TypeReference<Map<String, TableDefinition>>() {}),storageFactory.create("dynamodb", "dynamodb-items.json",new TypeReference<Map<String, Map<String, JsonNode>>>() {}),- regionResolver, streamService);+ regionResolver, streamService, kinesisForwarder);}/** Package-private constructor for testing. */DynamoDbService(StorageBackend<String, TableDefinition> tableStore) {- this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null);+ this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore, RegionResolver regionResolver) {- this(tableStore, null, regionResolver, null);+ this(tableStore, null, regionResolver, null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore,StorageBackend<String, Map<String, JsonNode>> itemStore,RegionResolver regionResolver) {- this(tableStore, itemStore, regionResolver, null);+ this(tableStore, itemStore, regionResolver, null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore,StorageBackend<String, Map<String, JsonNode>> itemStore,RegionResolver regionResolver,- DynamoDbStreamService streamService) {+ DynamoDbStreamService streamService,+ KinesisStreamingForwarder kinesisForwarder) {this.tableStore = tableStore;this.itemStore = itemStore;this.regionResolver = regionResolver;this.streamService = streamService;+ this.kinesisForwarder = kinesisForwarder;loadPersistedItems();}@@ -248,7 +252,9 @@ public class DynamoDbService {String eventName = existing == null ? "INSERT" : "MODIFY";if (streamService != null) {streamService.captureEvent(tableName, eventName, existing, item, table, region);- streamService.forwardToKinesisDestinations(eventName, existing, item, table, region);+ }+ if (kinesisForwarder != null) {+ kinesisForwarder.forward(eventName, existing, item, table, region);}}@@ -298,9 +304,13 @@ public class DynamoDbService {persistItems(storageKey);LOG.debugv("Deleted item from {0}: key={1}", tableName, itemKey);- if (streamService != null && removed != null) {- streamService.captureEvent(tableName, "REMOVE", removed, null, table, region);- streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);+ if (removed != null) {+ if (streamService != null) {+ streamService.captureEvent(tableName, "REMOVE", removed, null, table, region);+ }+ if (kinesisForwarder != null) {+ kinesisForwarder.forward("REMOVE", removed, null, table, region);+ }}return removed;@@ -377,7 +387,9 @@ public class DynamoDbService {if (streamService != null) {streamService.captureEvent(tableName, "MODIFY", existing, item, table, region);- streamService.forwardToKinesisDestinations("MODIFY", existing, item, table, region);+ }+ if (kinesisForwarder != null) {+ kinesisForwarder.forward("MODIFY", existing, item, table, region);}return new UpdateResult(item, existing);@@ -818,9 +830,13 @@ public class DynamoDbService {String region = storageKey.split("::", 2)[0];for (String itemKey : expiredKeys) {JsonNode removed = items.remove(itemKey);- if (removed != null && streamService != null) {- streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);- streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);+ if (removed != null) {+ if (streamService != null) {+ streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);+ }+ if (kinesisForwarder != null) {+ kinesisForwarder.forward("REMOVE", removed, null, table, region);+ }}}persistItems(storageKey);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.javaindex 11d6d2df0f..d5b3e9f3c4 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java@@ -5,10 +5,8 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;-import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;-import io.github.hectorvent.floci.services.kinesis.KinesisService;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;@@ -46,24 +44,16 @@ public class DynamoDbStreamService {private final AtomicLong sequenceCounter = new AtomicLong(0);private final ObjectMapper objectMapper;- private KinesisService kinesisService;@Inject- public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory,- KinesisService kinesisService) {+ public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory) {this(objectMapper, storageFactory.create("dynamodb", "dynamodb-tables.json",- new TypeReference<Map<String, TableDefinition>>() {}), kinesisService);+ new TypeReference<Map<String, TableDefinition>>() {}));}/** Package-private constructor for testing. */DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore) {- this(objectMapper, tableStore, null);- }-- DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore,- KinesisService kinesisService) {this.objectMapper = objectMapper;- this.kinesisService = kinesisService;loadPersistedStreams(tableStore);}@@ -175,85 +165,6 @@ public class DynamoDbStreamService {}}- public void forwardToKinesisDestinations(String eventName, JsonNode oldItem, JsonNode newItem,- TableDefinition table, String region) {- if (kinesisService == null) return;-- List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();- if (destinations == null || destinations.isEmpty()) return;-- Instant now = Instant.now();- JsonNode sourceItem = newItem != null ? newItem : oldItem;- ObjectNode keys = buildKeys(sourceItem, table);-- for (KinesisStreamingDestination dest : destinations) {- if (!"ACTIVE".equals(dest.getDestinationStatus())) continue;-- try {- ObjectNode payload = buildKinesisPayload(eventName, keys, newItem, oldItem,- table.getTableName(), region, now);- byte[] data = objectMapper.writeValueAsBytes(payload);-- String partitionKey = extractPartitionKey(keys, table);- String streamName = extractStreamName(dest.getStreamArn());-- kinesisService.putRecord(streamName, data, partitionKey, region);- LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",- streamName, eventName, table.getTableName());- } catch (Exception e) {- LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",- dest.getStreamArn(), e.getMessage());- }- }- }-- private ObjectNode buildKinesisPayload(String eventName, JsonNode keys,- JsonNode newImage, JsonNode oldImage,- String tableName, String region, Instant timestamp) {- ObjectNode payload = objectMapper.createObjectNode();- payload.put("awsRegion", region);- payload.put("eventID", UUID.randomUUID().toString());- payload.put("eventName", eventName);- payload.putNull("userIdentity");- payload.put("recordFormat", "application/json");- payload.put("tableName", tableName);- payload.put("eventSource", "aws:dynamodb");-- ObjectNode dynamodb = objectMapper.createObjectNode();- dynamodb.put("ApproximateCreationDateTime", timestamp.toEpochMilli());- if (keys != null) {- dynamodb.set("Keys", keys);- }- if (newImage != null) {- dynamodb.set("NewImage", newImage);- }- if (oldImage != null) {- dynamodb.set("OldImage", oldImage);- }- dynamodb.put("SizeBytes", 0);- dynamodb.put("ApproximateCreationDateTimePrecision", "MILLISECOND");- payload.set("dynamodb", dynamodb);-- return payload;- }-- private String extractPartitionKey(JsonNode keys, TableDefinition table) {- if (keys == null || keys.isEmpty()) return "default";- String pkName = table.getPartitionKeyName();- JsonNode pkValue = keys.get(pkName);- if (pkValue == null) return "default";- if (pkValue.has("S")) return pkValue.get("S").asText();- if (pkValue.has("N")) return pkValue.get("N").asText();- if (pkValue.has("B")) return pkValue.get("B").asText();- return pkValue.toString();- }-- private String extractStreamName(String streamArn) {- int idx = streamArn.lastIndexOf('/');- if (idx >= 0) return streamArn.substring(idx + 1);- return streamArn;- }-private ObjectNode buildKeys(JsonNode item, TableDefinition table) {ObjectNode keys = objectMapper.createObjectNode();if (item == null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.javanew file mode 100644index 0000000000..b10b9c97de--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java@@ -0,0 +1,120 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Instant;+import java.util.List;+import java.util.UUID;++@ApplicationScoped+public class KinesisStreamingForwarder {++ private static final Logger LOG = Logger.getLogger(KinesisStreamingForwarder.class);++ private final KinesisService kinesisService;+ private final ObjectMapper objectMapper;++ @Inject+ public KinesisStreamingForwarder(KinesisService kinesisService, ObjectMapper objectMapper) {+ this.kinesisService = kinesisService;+ this.objectMapper = objectMapper;+ }++ public void forward(String eventName, JsonNode oldItem, JsonNode newItem,+ TableDefinition table, String region) {+ List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();+ if (destinations == null || destinations.isEmpty()) return;++ Instant now = Instant.now();+ JsonNode sourceItem = newItem != null ? newItem : oldItem;+ ObjectNode keys = buildKeys(sourceItem, table);++ for (KinesisStreamingDestination dest : destinations) {+ if (!"ACTIVE".equals(dest.getDestinationStatus())) continue;++ try {+ ObjectNode payload = buildPayload(eventName, keys, newItem, oldItem,+ table.getTableName(), region, now);+ byte[] data = objectMapper.writeValueAsBytes(payload);++ String partitionKey = extractPartitionKey(keys, table);+ String streamName = extractStreamName(dest.getStreamArn());++ kinesisService.putRecord(streamName, data, partitionKey, region);+ LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",+ streamName, eventName, table.getTableName());+ } catch (Exception e) {+ LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",+ dest.getStreamArn(), e.getMessage());+ }+ }+ }++ private ObjectNode buildPayload(String eventName, JsonNode keys,+ JsonNode newImage, JsonNode oldImage,+ String tableName, String region, Instant timestamp) {+ ObjectNode payload = objectMapper.createObjectNode();+ payload.put("awsRegion", region);+ payload.put("eventID", UUID.randomUUID().toString());+ payload.put("eventName", eventName);+ payload.putNull("userIdentity");+ payload.put("recordFormat", "application/json");+ payload.put("tableName", tableName);+ payload.put("eventSource", "aws:dynamodb");++ ObjectNode dynamodb = objectMapper.createObjectNode();+ dynamodb.put("ApproximateCreationDateTime", timestamp.toEpochMilli());+ if (keys != null) {+ dynamodb.set("Keys", keys);+ }+ if (newImage != null) {+ dynamodb.set("NewImage", newImage);+ }+ if (oldImage != null) {+ dynamodb.set("OldImage", oldImage);+ }+ dynamodb.put("SizeBytes", 0);+ dynamodb.put("ApproximateCreationDateTimePrecision", "MILLISECOND");+ payload.set("dynamodb", dynamodb);++ return payload;+ }++ private ObjectNode buildKeys(JsonNode item, TableDefinition table) {+ ObjectNode keys = objectMapper.createObjectNode();+ if (item == null) return keys;+ for (KeySchemaElement ks : table.getKeySchema()) {+ String attrName = ks.getAttributeName();+ if (item.has(attrName)) {+ keys.set(attrName, item.get(attrName));+ }+ }+ return keys;+ }++ private String extractPartitionKey(JsonNode keys, TableDefinition table) {+ if (keys == null || keys.isEmpty()) return "default";+ String pkName = table.getPartitionKeyName();+ JsonNode pkValue = keys.get(pkName);+ if (pkValue == null) return "default";+ if (pkValue.has("S")) return pkValue.get("S").asText();+ if (pkValue.has("N")) return pkValue.get("N").asText();+ if (pkValue.has("B")) return pkValue.get("B").asText();+ return pkValue.toString();+ }++ private String extractStreamName(String streamArn) {+ int idx = streamArn.lastIndexOf('/');+ if (idx >= 0) return streamArn.substring(idx + 1);+ return streamArn;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.javanew file mode 100644index 0000000000..1b8a004626--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java@@ -0,0 +1,102 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;++import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++class KinesisStreamingForwarderTest {++ private KinesisService kinesisService;+ private KinesisStreamingForwarder forwarder;+ private ObjectMapper objectMapper;++ @BeforeEach+ void setUp() {+ kinesisService = mock(KinesisService.class);+ objectMapper = new ObjectMapper();+ forwarder = new KinesisStreamingForwarder(kinesisService, objectMapper);+ }++ private TableDefinition createTable(String tableName) {+ return new TableDefinition(tableName,+ List.of(new KeySchemaElement("pk", "HASH")),+ List.of(new AttributeDefinition("pk", "S")));+ }++ private ObjectNode createItem(String pk) {+ ObjectNode item = objectMapper.createObjectNode();+ ObjectNode pkValue = objectMapper.createObjectNode();+ pkValue.put("S", pk);+ item.set("pk", pkValue);+ return item;+ }++ @Test+ void forwardsToActiveDestination() {+ TableDefinition table = createTable("test-table");+ KinesisStreamingDestination dest = new KinesisStreamingDestination(+ "arn:aws:kinesis:us-east-1:000000000000:stream/test-stream");+ table.getKinesisStreamingDestinations().add(dest);++ when(kinesisService.putRecord(anyString(), any(byte[].class), anyString(), anyString()))+ .thenReturn("seq-1");++ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");++ verify(kinesisService).putRecord(eq("test-stream"), any(byte[].class), eq("k1"), eq("us-east-1"));+ }++ @Test+ void skipsDisabledDestination() {+ TableDefinition table = createTable("test-table");+ KinesisStreamingDestination dest = new KinesisStreamingDestination(+ "arn:aws:kinesis:us-east-1:000000000000:stream/test-stream");+ dest.setDestinationStatus("DISABLED");+ table.getKinesisStreamingDestinations().add(dest);++ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");++ verifyNoInteractions(kinesisService);+ }++ @Test+ void skipsWhenNoDestinations() {+ TableDefinition table = createTable("test-table");++ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");++ verifyNoInteractions(kinesisService);+ }++ @Test+ void continuesOnPutRecordFailure() {+ TableDefinition table = createTable("test-table");+ KinesisStreamingDestination dest1 = new KinesisStreamingDestination(+ "arn:aws:kinesis:us-east-1:000000000000:stream/stream-1");+ KinesisStreamingDestination dest2 = new KinesisStreamingDestination(+ "arn:aws:kinesis:us-east-1:000000000000:stream/stream-2");+ table.getKinesisStreamingDestinations().add(dest1);+ table.getKinesisStreamingDestinations().add(dest2);++ when(kinesisService.putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString()))+ .thenThrow(new RuntimeException("stream-1 failed"));+ when(kinesisService.putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString()))+ .thenReturn("seq-1");++ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");++ verify(kinesisService).putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString());+ verify(kinesisService).putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString());+ }+}