Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

refactor(dynamodb): extract KinesisStreamingForwarder from DynamoDbStreamService (#428)

yusuke-uranakaa0cd0ac511b4
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 0c9f37077e..02a5730a3f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -38,40 +38,44 @@ public class DynamoDbService {
private final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();
private final RegionResolver regionResolver;
private DynamoDbStreamService streamService;
+ private KinesisStreamingForwarder kinesisForwarder;
@Inject
public DynamoDbService(StorageFactory storageFactory, RegionResolver regionResolver,
- DynamoDbStreamService streamService) {
+ DynamoDbStreamService streamService,
+ KinesisStreamingForwarder kinesisForwarder) {
this(storageFactory.create("dynamodb", "dynamodb-tables.json",
new TypeReference<Map<String, TableDefinition>>() {}),
storageFactory.create("dynamodb", "dynamodb-items.json",
new TypeReference<Map<String, Map<String, JsonNode>>>() {}),
- regionResolver, streamService);
+ regionResolver, streamService, kinesisForwarder);
}
/** Package-private constructor for testing. */
DynamoDbService(StorageBackend<String, TableDefinition> tableStore) {
- this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null);
+ this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore, RegionResolver regionResolver) {
- this(tableStore, null, regionResolver, null);
+ this(tableStore, null, regionResolver, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
StorageBackend<String, Map<String, JsonNode>> itemStore,
RegionResolver regionResolver) {
- this(tableStore, itemStore, regionResolver, null);
+ this(tableStore, itemStore, regionResolver, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
StorageBackend<String, Map<String, JsonNode>> itemStore,
RegionResolver regionResolver,
- DynamoDbStreamService streamService) {
+ DynamoDbStreamService streamService,
+ KinesisStreamingForwarder kinesisForwarder) {
this.tableStore = tableStore;
this.itemStore = itemStore;
this.regionResolver = regionResolver;
this.streamService = streamService;
+ this.kinesisForwarder = kinesisForwarder;
loadPersistedItems();
}
@@ -248,7 +252,9 @@ public class DynamoDbService {
String eventName = existing == null ? "INSERT" : "MODIFY";
if (streamService != null) {
streamService.captureEvent(tableName, eventName, existing, item, table, region);
- streamService.forwardToKinesisDestinations(eventName, existing, item, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward(eventName, existing, item, table, region);
}
}
@@ -298,9 +304,13 @@ public class DynamoDbService {
persistItems(storageKey);
LOG.debugv("Deleted item from {0}: key={1}", tableName, itemKey);
- if (streamService != null && removed != null) {
- streamService.captureEvent(tableName, "REMOVE", removed, null, table, region);
- streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);
+ if (removed != null) {
+ if (streamService != null) {
+ streamService.captureEvent(tableName, "REMOVE", removed, null, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward("REMOVE", removed, null, table, region);
+ }
}
return removed;
@@ -377,7 +387,9 @@ public class DynamoDbService {
if (streamService != null) {
streamService.captureEvent(tableName, "MODIFY", existing, item, table, region);
- streamService.forwardToKinesisDestinations("MODIFY", existing, item, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward("MODIFY", existing, item, table, region);
}
return new UpdateResult(item, existing);
@@ -818,9 +830,13 @@ public class DynamoDbService {
String region = storageKey.split("::", 2)[0];
for (String itemKey : expiredKeys) {
JsonNode removed = items.remove(itemKey);
- if (removed != null && streamService != null) {
- streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);
- streamService.forwardToKinesisDestinations("REMOVE", removed, null, table, region);
+ if (removed != null) {
+ if (streamService != null) {
+ streamService.captureEvent(table.getTableName(), "REMOVE", removed, null, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward("REMOVE", removed, null, table, region);
+ }
}
}
persistItems(storageKey);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index 11d6d2df0f..d5b3e9f3c4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -5,10 +5,8 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
-import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
-import io.github.hectorvent.floci.services.kinesis.KinesisService;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -46,24 +44,16 @@ public class DynamoDbStreamService {
private final AtomicLong sequenceCounter = new AtomicLong(0);
private final ObjectMapper objectMapper;
- private KinesisService kinesisService;
@Inject
- public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory,
- KinesisService kinesisService) {
+ public DynamoDbStreamService(ObjectMapper objectMapper, StorageFactory storageFactory) {
this(objectMapper, storageFactory.create("dynamodb", "dynamodb-tables.json",
- new TypeReference<Map<String, TableDefinition>>() {}), kinesisService);
+ new TypeReference<Map<String, TableDefinition>>() {}));
}
/** Package-private constructor for testing. */
DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore) {
- this(objectMapper, tableStore, null);
- }
-
- DynamoDbStreamService(ObjectMapper objectMapper, StorageBackend<String, TableDefinition> tableStore,
- KinesisService kinesisService) {
this.objectMapper = objectMapper;
- this.kinesisService = kinesisService;
loadPersistedStreams(tableStore);
}
@@ -175,85 +165,6 @@ public class DynamoDbStreamService {
}
}
- public void forwardToKinesisDestinations(String eventName, JsonNode oldItem, JsonNode newItem,
- TableDefinition table, String region) {
- if (kinesisService == null) return;
-
- List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();
- if (destinations == null || destinations.isEmpty()) return;
-
- Instant now = Instant.now();
- JsonNode sourceItem = newItem != null ? newItem : oldItem;
- ObjectNode keys = buildKeys(sourceItem, table);
-
- for (KinesisStreamingDestination dest : destinations) {
- if (!"ACTIVE".equals(dest.getDestinationStatus())) continue;
-
- try {
- ObjectNode payload = buildKinesisPayload(eventName, keys, newItem, oldItem,
- table.getTableName(), region, now);
- byte[] data = objectMapper.writeValueAsBytes(payload);
-
- String partitionKey = extractPartitionKey(keys, table);
- String streamName = extractStreamName(dest.getStreamArn());
-
- kinesisService.putRecord(streamName, data, partitionKey, region);
- LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",
- streamName, eventName, table.getTableName());
- } catch (Exception e) {
- LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",
- dest.getStreamArn(), e.getMessage());
- }
- }
- }
-
- private ObjectNode buildKinesisPayload(String eventName, JsonNode keys,
- JsonNode newImage, JsonNode oldImage,
- String tableName, String region, Instant timestamp) {
- ObjectNode payload = objectMapper.createObjectNode();
- payload.put("awsRegion", region);
- payload.put("eventID", UUID.randomUUID().toString());
- payload.put("eventName", eventName);
- payload.putNull("userIdentity");
- payload.put("recordFormat", "application/json");
- payload.put("tableName", tableName);
- payload.put("eventSource", "aws:dynamodb");
-
- ObjectNode dynamodb = objectMapper.createObjectNode();
- dynamodb.put("ApproximateCreationDateTime", timestamp.toEpochMilli());
- if (keys != null) {
- dynamodb.set("Keys", keys);
- }
- if (newImage != null) {
- dynamodb.set("NewImage", newImage);
- }
- if (oldImage != null) {
- dynamodb.set("OldImage", oldImage);
- }
- dynamodb.put("SizeBytes", 0);
- dynamodb.put("ApproximateCreationDateTimePrecision", "MILLISECOND");
- payload.set("dynamodb", dynamodb);
-
- return payload;
- }
-
- private String extractPartitionKey(JsonNode keys, TableDefinition table) {
- if (keys == null || keys.isEmpty()) return "default";
- String pkName = table.getPartitionKeyName();
- JsonNode pkValue = keys.get(pkName);
- if (pkValue == null) return "default";
- if (pkValue.has("S")) return pkValue.get("S").asText();
- if (pkValue.has("N")) return pkValue.get("N").asText();
- if (pkValue.has("B")) return pkValue.get("B").asText();
- return pkValue.toString();
- }
-
- private String extractStreamName(String streamArn) {
- int idx = streamArn.lastIndexOf('/');
- if (idx >= 0) return streamArn.substring(idx + 1);
- return streamArn;
- }
-
private ObjectNode buildKeys(JsonNode item, TableDefinition table) {
ObjectNode keys = objectMapper.createObjectNode();
if (item == null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
new file mode 100644
index 0000000000..b10b9c97de
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
@@ -0,0 +1,120 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+@ApplicationScoped
+public class KinesisStreamingForwarder {
+
+ private static final Logger LOG = Logger.getLogger(KinesisStreamingForwarder.class);
+
+ private final KinesisService kinesisService;
+ private final ObjectMapper objectMapper;
+
+ @Inject
+ public KinesisStreamingForwarder(KinesisService kinesisService, ObjectMapper objectMapper) {
+ this.kinesisService = kinesisService;
+ this.objectMapper = objectMapper;
+ }
+
+ public void forward(String eventName, JsonNode oldItem, JsonNode newItem,
+ TableDefinition table, String region) {
+ List<KinesisStreamingDestination> destinations = table.getKinesisStreamingDestinations();
+ if (destinations == null || destinations.isEmpty()) return;
+
+ Instant now = Instant.now();
+ JsonNode sourceItem = newItem != null ? newItem : oldItem;
+ ObjectNode keys = buildKeys(sourceItem, table);
+
+ for (KinesisStreamingDestination dest : destinations) {
+ if (!"ACTIVE".equals(dest.getDestinationStatus())) continue;
+
+ try {
+ ObjectNode payload = buildPayload(eventName, keys, newItem, oldItem,
+ table.getTableName(), region, now);
+ byte[] data = objectMapper.writeValueAsBytes(payload);
+
+ String partitionKey = extractPartitionKey(keys, table);
+ String streamName = extractStreamName(dest.getStreamArn());
+
+ kinesisService.putRecord(streamName, data, partitionKey, region);
+ LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",
+ streamName, eventName, table.getTableName());
+ } catch (Exception e) {
+ LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",
+ dest.getStreamArn(), e.getMessage());
+ }
+ }
+ }
+
+ private ObjectNode buildPayload(String eventName, JsonNode keys,
+ JsonNode newImage, JsonNode oldImage,
+ String tableName, String region, Instant timestamp) {
+ ObjectNode payload = objectMapper.createObjectNode();
+ payload.put("awsRegion", region);
+ payload.put("eventID", UUID.randomUUID().toString());
+ payload.put("eventName", eventName);
+ payload.putNull("userIdentity");
+ payload.put("recordFormat", "application/json");
+ payload.put("tableName", tableName);
+ payload.put("eventSource", "aws:dynamodb");
+
+ ObjectNode dynamodb = objectMapper.createObjectNode();
+ dynamodb.put("ApproximateCreationDateTime", timestamp.toEpochMilli());
+ if (keys != null) {
+ dynamodb.set("Keys", keys);
+ }
+ if (newImage != null) {
+ dynamodb.set("NewImage", newImage);
+ }
+ if (oldImage != null) {
+ dynamodb.set("OldImage", oldImage);
+ }
+ dynamodb.put("SizeBytes", 0);
+ dynamodb.put("ApproximateCreationDateTimePrecision", "MILLISECOND");
+ payload.set("dynamodb", dynamodb);
+
+ return payload;
+ }
+
+ private ObjectNode buildKeys(JsonNode item, TableDefinition table) {
+ ObjectNode keys = objectMapper.createObjectNode();
+ if (item == null) return keys;
+ for (KeySchemaElement ks : table.getKeySchema()) {
+ String attrName = ks.getAttributeName();
+ if (item.has(attrName)) {
+ keys.set(attrName, item.get(attrName));
+ }
+ }
+ return keys;
+ }
+
+ private String extractPartitionKey(JsonNode keys, TableDefinition table) {
+ if (keys == null || keys.isEmpty()) return "default";
+ String pkName = table.getPartitionKeyName();
+ JsonNode pkValue = keys.get(pkName);
+ if (pkValue == null) return "default";
+ if (pkValue.has("S")) return pkValue.get("S").asText();
+ if (pkValue.has("N")) return pkValue.get("N").asText();
+ if (pkValue.has("B")) return pkValue.get("B").asText();
+ return pkValue.toString();
+ }
+
+ private String extractStreamName(String streamArn) {
+ int idx = streamArn.lastIndexOf('/');
+ if (idx >= 0) return streamArn.substring(idx + 1);
+ return streamArn;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java
new file mode 100644
index 0000000000..1b8a004626
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java
@@ -0,0 +1,102 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+class KinesisStreamingForwarderTest {
+
+ private KinesisService kinesisService;
+ private KinesisStreamingForwarder forwarder;
+ private ObjectMapper objectMapper;
+
+ @BeforeEach
+ void setUp() {
+ kinesisService = mock(KinesisService.class);
+ objectMapper = new ObjectMapper();
+ forwarder = new KinesisStreamingForwarder(kinesisService, objectMapper);
+ }
+
+ private TableDefinition createTable(String tableName) {
+ return new TableDefinition(tableName,
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")));
+ }
+
+ private ObjectNode createItem(String pk) {
+ ObjectNode item = objectMapper.createObjectNode();
+ ObjectNode pkValue = objectMapper.createObjectNode();
+ pkValue.put("S", pk);
+ item.set("pk", pkValue);
+ return item;
+ }
+
+ @Test
+ void forwardsToActiveDestination() {
+ TableDefinition table = createTable("test-table");
+ KinesisStreamingDestination dest = new KinesisStreamingDestination(
+ "arn:aws:kinesis:us-east-1:000000000000:stream/test-stream");
+ table.getKinesisStreamingDestinations().add(dest);
+
+ when(kinesisService.putRecord(anyString(), any(byte[].class), anyString(), anyString()))
+ .thenReturn("seq-1");
+
+ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");
+
+ verify(kinesisService).putRecord(eq("test-stream"), any(byte[].class), eq("k1"), eq("us-east-1"));
+ }
+
+ @Test
+ void skipsDisabledDestination() {
+ TableDefinition table = createTable("test-table");
+ KinesisStreamingDestination dest = new KinesisStreamingDestination(
+ "arn:aws:kinesis:us-east-1:000000000000:stream/test-stream");
+ dest.setDestinationStatus("DISABLED");
+ table.getKinesisStreamingDestinations().add(dest);
+
+ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");
+
+ verifyNoInteractions(kinesisService);
+ }
+
+ @Test
+ void skipsWhenNoDestinations() {
+ TableDefinition table = createTable("test-table");
+
+ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");
+
+ verifyNoInteractions(kinesisService);
+ }
+
+ @Test
+ void continuesOnPutRecordFailure() {
+ TableDefinition table = createTable("test-table");
+ KinesisStreamingDestination dest1 = new KinesisStreamingDestination(
+ "arn:aws:kinesis:us-east-1:000000000000:stream/stream-1");
+ KinesisStreamingDestination dest2 = new KinesisStreamingDestination(
+ "arn:aws:kinesis:us-east-1:000000000000:stream/stream-2");
+ table.getKinesisStreamingDestinations().add(dest1);
+ table.getKinesisStreamingDestinations().add(dest2);
+
+ when(kinesisService.putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString()))
+ .thenThrow(new RuntimeException("stream-1 failed"));
+ when(kinesisService.putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString()))
+ .thenReturn("seq-1");
+
+ forwarder.forward("INSERT", null, createItem("k1"), table, "us-east-1");
+
+ verify(kinesisService).putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString());
+ verify(kinesisService).putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString());
+ }
+}