Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): flush persistence once per affected table in batch and transact writes (#2998)

Vũ Thế Huy80725e6f483a
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 45dbed8d10..7f3ac0d8f6 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -55,6 +55,14 @@ DynamoDB Streams are supported via a separate target (`DynamoDBStreams_20120810`
| `FLOCI_STORAGE_SERVICES_DYNAMODB_MODE` | *(global default)* | Storage mode override for DynamoDB (`memory`, `persistent`, `hybrid`, `wal`) |
| `FLOCI_STORAGE_SERVICES_DYNAMODB_FLUSH_INTERVAL_MS` | `5000` | Flush interval for `hybrid`/`wal` storage modes (milliseconds) |
+### Storage and Performance
+
+Under `persistent` storage mode, single-item writes (`PutItem`, `UpdateItem`, `DeleteItem`) flush the affected table to disk synchronously. Batch operations (`BatchWriteItem` and `TransactWriteItems`) batch disk flushes per affected table across the entire operation, rather than flushing on every individual item mutation.
+
+For write-heavy workloads under persistent setups, configuring `FLOCI_STORAGE_SERVICES_DYNAMODB_MODE=wal` or `hybrid` is recommended to avoid full-file rewrites on each write operation:
+- `wal`: Uses an append-only write-ahead log with background compaction.
+- `hybrid`: Keeps data in memory with periodic asynchronous disk flushes controlled by `FLOCI_STORAGE_SERVICES_DYNAMODB_FLUSH_INTERVAL_MS`.
+
## Examples
```bash
@@ -156,5 +164,4 @@ aws dynamodb list-exports \
--endpoint-url $AWS_ENDPOINT_URL
```
-The export writes to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/data/` as one or more `.json.gz` files, along with `manifest-summary.json` and `manifest-files.json` — the same layout as real AWS DynamoDB exports.
-```
\ No newline at end of file
+The export writes to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/data/` as one or more `.json.gz` files, along with `manifest-summary.json` and `manifest-files.json`, the same layout as real AWS DynamoDB exports.
\ No newline at end of file
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 6a02aea0b1..a0dbf13b70 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -20,6 +20,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -27,9 +29,11 @@ import org.jboss.logging.Logger;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@@ -47,7 +51,9 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import java.util.zip.GZIPOutputStream;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
@@ -89,7 +95,7 @@ public class DynamoDbService implements ResourceProvider {
// request body so a replay with the same token but different parameters can be
// rejected with IdempotentParameterMismatchException.
private final ConcurrentHashMap<String, IdempotencyEntry> txIdempotency = new ConcurrentHashMap<>();
- private static final long TX_IDEMPOTENCY_TTL_NANOS = java.time.Duration.ofMinutes(10).toNanos();
+ private static final long TX_IDEMPOTENCY_TTL_NANOS = Duration.ofMinutes(10).toNanos();
private static final int MAX_MULTI_ATTRIBUTE_KEY_PART_SIZE = 4;
@@ -238,7 +244,7 @@ public class DynamoDbService implements ResourceProvider {
if (keySchema.size() > 2) {
String repr = "[" + keySchema.stream()
.map(k -> "KeySchemaElement(attributeName=" + k.getAttributeName() + ", keyType=" + k.getKeyType() + ")")
- .collect(java.util.stream.Collectors.joining(", ")) + "]";
+ .collect(Collectors.joining(", ")) + "]";
throw new AwsException("ValidationException",
"1 validation error detected: Value '" + repr + "' at 'keySchema' failed to satisfy constraint: "
+ "Member must have length less than or equal to 2", 400);
@@ -278,7 +284,7 @@ public class DynamoDbService implements ResourceProvider {
? Set.of()
: attributeDefinitions.stream()
.map(AttributeDefinition::getAttributeName)
- .collect(java.util.stream.Collectors.toSet());
+ .collect(Collectors.toSet());
if (!definedAttrs.containsAll(referencedAttrs)) {
throw new AwsException("ValidationException",
"Invalid KeySchema: Some index key attribute have no definition", 400);
@@ -503,6 +509,25 @@ public class DynamoDbService implements ResourceProvider {
String conditionExpression,
JsonNode exprAttrNames, JsonNode exprAttrValues,
String region, String returnValuesOnConditionCheckFailure) {
+ putItemInternal(tableName, item, conditionExpression, exprAttrNames, exprAttrValues,
+ region, returnValuesOnConditionCheckFailure, true);
+ }
+
+ private JsonNode putItemInternal(String tableName, JsonNode item,
+ String conditionExpression,
+ JsonNode exprAttrNames, JsonNode exprAttrValues,
+ String region, String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist) {
+ return putItemInternal(tableName, item, conditionExpression, exprAttrNames, exprAttrValues,
+ region, returnValuesOnConditionCheckFailure, shouldPersist, null);
+ }
+
+ private JsonNode putItemInternal(String tableName, JsonNode item,
+ String conditionExpression,
+ JsonNode exprAttrNames, JsonNode exprAttrValues,
+ String region, String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist,
+ Consumer<Runnable> deferredStreamEvents) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
@@ -514,7 +539,7 @@ public class DynamoDbService implements ResourceProvider {
String itemKey = buildItemKey(table, normalizedItem);
validateIndexKeyTypes(table, normalizedItem, false);
- withItemLock(storageKey, itemKey, () -> {
+ return withItemLock(storageKey, itemKey, () -> {
var tableItems = itemsByTable.computeIfAbsent(scopedItemsKey(storageKey), k -> new ConcurrentSkipListMap<>());
JsonNode existing = tableItems.get(itemKey);
@@ -524,17 +549,27 @@ public class DynamoDbService implements ResourceProvider {
}
tableItems.put(itemKey, normalizedItem);
- persistItems(storageKey);
+ if (shouldPersist) {
+ persistItems(storageKey);
+ }
LOG.debugv("Put item in {0}: key={1}", canonicalTableName, itemKey);
LOG.tracev("Put item in {0}: key={1} item={2}", canonicalTableName, itemKey, item);
String eventName = existing == null ? "INSERT" : "MODIFY";
- if (streamService != null) {
- streamService.captureEvent(canonicalTableName, eventName, existing, item, table, region);
- }
- if (kinesisForwarder != null) {
- kinesisForwarder.forward(eventName, existing, item, table, region);
+ Runnable streamEvent = () -> {
+ if (streamService != null) {
+ streamService.captureEvent(canonicalTableName, eventName, existing, item, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward(eventName, existing, item, table, region);
+ }
+ };
+ if (deferredStreamEvents != null) {
+ deferredStreamEvents.accept(streamEvent);
+ } else {
+ streamEvent.run();
}
+ return existing;
});
}
@@ -567,6 +602,25 @@ public class DynamoDbService implements ResourceProvider {
String conditionExpression,
JsonNode exprAttrNames, JsonNode exprAttrValues,
String region, String returnValuesOnConditionCheckFailure) {
+ return deleteItemInternal(tableName, key, conditionExpression, exprAttrNames, exprAttrValues,
+ region, returnValuesOnConditionCheckFailure, true);
+ }
+
+ private JsonNode deleteItemInternal(String tableName, JsonNode key,
+ String conditionExpression,
+ JsonNode exprAttrNames, JsonNode exprAttrValues,
+ String region, String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist) {
+ return deleteItemInternal(tableName, key, conditionExpression, exprAttrNames, exprAttrValues,
+ region, returnValuesOnConditionCheckFailure, shouldPersist, null);
+ }
+
+ private JsonNode deleteItemInternal(String tableName, JsonNode key,
+ String conditionExpression,
+ JsonNode exprAttrNames, JsonNode exprAttrValues,
+ String region, String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist,
+ Consumer<Runnable> deferredStreamEvents) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
@@ -584,16 +638,25 @@ public class DynamoDbService implements ResourceProvider {
}
JsonNode removed = items.remove(itemKey);
- persistItems(storageKey);
+ if (shouldPersist) {
+ persistItems(storageKey);
+ }
LOG.debugv("Deleted item from {0}: key={1}", canonicalTableName, itemKey);
LOG.tracev("Deleted item from {0}: key={1} removed={2}", canonicalTableName, itemKey, removed);
if (removed != null) {
- if (streamService != null) {
- streamService.captureEvent(canonicalTableName, "REMOVE", removed, null, table, region);
- }
- if (kinesisForwarder != null) {
- kinesisForwarder.forward("REMOVE", removed, null, table, region);
+ Runnable streamEvent = () -> {
+ if (streamService != null) {
+ streamService.captureEvent(canonicalTableName, "REMOVE", removed, null, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward("REMOVE", removed, null, table, region);
+ }
+ };
+ if (deferredStreamEvents != null) {
+ deferredStreamEvents.accept(streamEvent);
+ } else {
+ streamEvent.run();
}
}
@@ -614,6 +677,29 @@ public class DynamoDbService implements ResourceProvider {
JsonNode expressionAttrNames, JsonNode expressionAttrValues,
String returnValues, String conditionExpression, String region,
String returnValuesOnConditionCheckFailure) {
+ return updateItemInternal(tableName, key, attributeUpdates, updateExpression,
+ expressionAttrNames, expressionAttrValues, returnValues,
+ conditionExpression, region, returnValuesOnConditionCheckFailure, true);
+ }
+
+ private UpdateResult updateItemInternal(String tableName, JsonNode key, JsonNode attributeUpdates,
+ String updateExpression,
+ JsonNode expressionAttrNames, JsonNode expressionAttrValues,
+ String returnValues, String conditionExpression, String region,
+ String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist) {
+ return updateItemInternal(tableName, key, attributeUpdates, updateExpression,
+ expressionAttrNames, expressionAttrValues, returnValues,
+ conditionExpression, region, returnValuesOnConditionCheckFailure, shouldPersist, null);
+ }
+
+ private UpdateResult updateItemInternal(String tableName, JsonNode key, JsonNode attributeUpdates,
+ String updateExpression,
+ JsonNode expressionAttrNames, JsonNode expressionAttrValues,
+ String returnValues, String conditionExpression, String region,
+ String returnValuesOnConditionCheckFailure,
+ boolean shouldPersist,
+ Consumer<Runnable> deferredStreamEvents) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
@@ -665,14 +751,14 @@ public class DynamoDbService implements ResourceProvider {
if (curAttr.has(setType) && value.has(setType)) {
Set<String> removeSet = new HashSet<>();
for (JsonNode v : value.get(setType)) removeSet.add(v.asText());
- com.fasterxml.jackson.databind.node.ArrayNode newArr = objectMapper.createArrayNode();
+ ArrayNode newArr = objectMapper.createArrayNode();
for (JsonNode v : curAttr.get(setType)) {
if (!removeSet.contains(v.asText())) newArr.add(v);
}
if (newArr.isEmpty()) {
item.remove(attrName);
} else {
- ((com.fasterxml.jackson.databind.node.ObjectNode) curAttr).set(setType, newArr);
+ ((ObjectNode) curAttr).set(setType, newArr);
}
break;
}
@@ -684,10 +770,10 @@ public class DynamoDbService implements ResourceProvider {
if (value != null) {
JsonNode curAttr = item.get(attrName);
if (value.has("N")) {
- java.math.BigDecimal delta = new java.math.BigDecimal(value.get("N").asText());
- java.math.BigDecimal current = curAttr != null && curAttr.has("N")
- ? new java.math.BigDecimal(curAttr.get("N").asText()) : java.math.BigDecimal.ZERO;
- com.fasterxml.jackson.databind.node.ObjectNode numNode = objectMapper.createObjectNode();
+ BigDecimal delta = new BigDecimal(value.get("N").asText());
+ BigDecimal current = curAttr != null && curAttr.has("N")
+ ? new BigDecimal(curAttr.get("N").asText()) : BigDecimal.ZERO;
+ ObjectNode numNode = objectMapper.createObjectNode();
numNode.put("N", current.add(delta).stripTrailingZeros().toPlainString());
item.set(attrName, numNode);
} else {
@@ -700,9 +786,9 @@ public class DynamoDbService implements ResourceProvider {
Set<String> existingSet = new LinkedHashSet<>();
for (JsonNode v : curAttr.get(setType)) existingSet.add(v.asText());
for (JsonNode v : value.get(setType)) existingSet.add(v.asText());
- com.fasterxml.jackson.databind.node.ArrayNode newArr = objectMapper.createArrayNode();
+ ArrayNode newArr = objectMapper.createArrayNode();
for (String s : existingSet) newArr.add(s);
- ((com.fasterxml.jackson.databind.node.ObjectNode) curAttr).set(setType, newArr);
+ ((ObjectNode) curAttr).set(setType, newArr);
}
break;
}
@@ -715,38 +801,30 @@ public class DynamoDbService implements ResourceProvider {
}
// Reject any attempt to modify a key attribute
- String pkName = table.getPartitionKeyName();
- JsonNode origPk = key.get(pkName);
- JsonNode newPk = item.get(pkName);
- if (origPk != null && newPk != null && !origPk.equals(newPk)) {
- throw new AwsException("ValidationException",
- "One or more parameter values were invalid: Cannot update attribute " + pkName
- + ". This attribute is part of the key", 400);
- }
- String skName = table.getSortKeyName();
- if (skName != null) {
- JsonNode origSk = key.get(skName);
- JsonNode newSk = item.get(skName);
- if (origSk != null && newSk != null && !origSk.equals(newSk)) {
- throw new AwsException("ValidationException",
- "One or more parameter values were invalid: Cannot update attribute " + skName
- + ". This attribute is part of the key", 400);
- }
- }
+ validateKeyNotModified(table, key, item);
// AWS validates index key values against the item the update produces.
validateIndexKeyTypes(table, item, true);
items.put(itemKey, item);
- persistItems(storageKey);
+ if (shouldPersist) {
+ persistItems(storageKey);
+ }
LOG.tracev("Updated item in {0}: key={1} updateExpression={2} item={3}",
canonicalTableName, itemKey, updateExpression, item);
- if (streamService != null) {
- streamService.captureEvent(canonicalTableName, "MODIFY", existing, item, table, region);
- }
- if (kinesisForwarder != null) {
- kinesisForwarder.forward("MODIFY", existing, item, table, region);
+ Runnable streamEvent = () -> {
+ if (streamService != null) {
+ streamService.captureEvent(canonicalTableName, "MODIFY", existing, item, table, region);
+ }
+ if (kinesisForwarder != null) {
+ kinesisForwarder.forward("MODIFY", existing, item, table, region);
+ }
+ };
+ if (deferredStreamEvents != null) {
+ deferredStreamEvents.accept(streamEvent);
+ } else {
+ streamEvent.run();
}
return new UpdateResult(item, existing);
@@ -1032,18 +1110,62 @@ public class DynamoDbService implements ResourceProvider {
public record BatchWriteResult(Map<String, List<JsonNode>> unprocessedItems) {}
public BatchWriteResult batchWriteItem(Map<String, List<JsonNode>> requestItems, String region) {
+ // Pre-validate all write requests before applying any mutation to memory or streams
for (Map.Entry<String, List<JsonNode>> entry : requestItems.entrySet()) {
String tableName = canonicalTableName(region, entry.getKey());
+ String storageKey = regionKey(region, tableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(tableName));
+ Set<String> seenKeys = new HashSet<>();
for (JsonNode writeRequest : entry.getValue()) {
+ String itemKey;
if (writeRequest.has("PutRequest")) {
JsonNode item = writeRequest.get("PutRequest").get("Item");
- putItem(tableName, item, region);
+ if (item == null) {
+ throw new AwsException("ValidationException", "Item is required for PutRequest", 400);
+ }
+ JsonNode normalizedItem = DynamoDbNumberUtils.normalizeNumbersInItem(item);
+ DynamoDbItemSize.validateSize(normalizedItem);
+ itemKey = buildItemKey(table, normalizedItem);
+ validateIndexKeyTypes(table, normalizedItem, false);
} else if (writeRequest.has("DeleteRequest")) {
JsonNode key = writeRequest.get("DeleteRequest").get("Key");
- deleteItem(tableName, key, region);
+ if (key == null) {
+ throw new AwsException("ValidationException", "Key is required for DeleteRequest", 400);
+ }
+ itemKey = buildItemKey(table, key, true);
+ } else {
+ continue;
+ }
+ if (!seenKeys.add(itemKey)) {
+ throw new AwsException("ValidationException",
+ "Provided list of item keys contains duplicates", 400);
}
}
}
+
+ Set<String> affectedStorageKeys = new LinkedHashSet<>();
+ try {
+ for (Map.Entry<String, List<JsonNode>> entry : requestItems.entrySet()) {
+ String tableName = canonicalTableName(region, entry.getKey());
+ String storageKey = regionKey(region, tableName);
+ for (JsonNode writeRequest : entry.getValue()) {
+ if (writeRequest.has("PutRequest")) {
+ JsonNode item = writeRequest.get("PutRequest").get("Item");
+ putItemInternal(tableName, item, null, null, null, region, "NONE", false);
+ affectedStorageKeys.add(storageKey);
+ } else if (writeRequest.has("DeleteRequest")) {
+ JsonNode key = writeRequest.get("DeleteRequest").get("Key");
+ deleteItemInternal(tableName, key, null, null, null, region, "NONE", false);
+ affectedStorageKeys.add(storageKey);
+ }
+ }
+ }
+ } finally {
+ for (String storageKey : affectedStorageKeys) {
+ persistItems(storageKey);
+ }
+ }
return new BatchWriteResult(Map.of());
}
@@ -1136,9 +1258,14 @@ public class DynamoDbService implements ResourceProvider {
// bytes in an item's PK/SK value cannot collide two distinct participants
// into the same ordering key.
TreeMap<TransactParticipant, ReentrantLock> toAcquire = new TreeMap<>(PARTICIPANT_ORDER);
+ Set<TransactParticipant> seenParticipants = new HashSet<>();
for (JsonNode transactItem : transactItems) {
TransactParticipant p = resolveParticipant(transactItem, region);
if (p == null) continue;
+ if (!seenParticipants.add(p)) {
+ throw new AwsException("ValidationException",
+ "Transaction request cannot include multiple operations on one item", 400);
+ }
toAcquire.putIfAbsent(p, lockFor(p.storageKey, p.itemKey));
}
@@ -1166,31 +1293,84 @@ public class DynamoDbService implements ResourceProvider {
throw new TransactionCanceledException(cancellationReasons);
}
- // Second pass: apply all writes. Inner methods re-acquire their own locks,
- // which is a no-op thanks to ReentrantLock.
+ // Pre-validation pass: validate all operations before mutating memory or emitting stream effects
for (JsonNode transactItem : transactItems) {
- if (transactItem.has("Put")) {
- JsonNode put = transactItem.get("Put");
- String tableName = put.path("TableName").asText();
- JsonNode item = put.get("Item");
- putItem(tableName, item, region);
- } else if (transactItem.has("Delete")) {
- JsonNode del = transactItem.get("Delete");
- String tableName = del.path("TableName").asText();
- JsonNode key = del.get("Key");
- deleteItem(tableName, key, region);
- } else if (transactItem.has("Update")) {
- JsonNode upd = transactItem.get("Update");
- String tableName = upd.path("TableName").asText();
- JsonNode key = upd.get("Key");
- String updateExpression = upd.has("UpdateExpression") ? upd.get("UpdateExpression").asText() : null;
- JsonNode exprAttrNames = upd.has("ExpressionAttributeNames") ? upd.get("ExpressionAttributeNames") : null;
- JsonNode exprAttrValues = upd.has("ExpressionAttributeValues") ? upd.get("ExpressionAttributeValues") : null;
- //there is no ConditionExpression, so setting returnValuesOnConditionCheckFailure = "NONE"
- updateItem(tableName, key, null, updateExpression, exprAttrNames, exprAttrValues,
- "NONE", null, region, "NONE");
+ validateTransactItem(transactItem, region);
+ }
+
+ // Second pass: apply all writes with rollback protection and deferred streams
+ record RollbackEntry(String storageKey, String itemKey, JsonNode previousValue) {}
+ List<RollbackEntry> rollbackList = new ArrayList<>();
+ List<Runnable> pendingStreamEvents = new ArrayList<>();
+ Set<String> affectedStorageKeys = new LinkedHashSet<>();
+ boolean transactionSucceeded = false;
+ try {
+ for (JsonNode transactItem : transactItems) {
+ if (transactItem.has("Put")) {
+ JsonNode put = transactItem.get("Put");
+ String tableName = put.path("TableName").asText();
+ JsonNode item = put.get("Item");
+ String canonicalTableName = canonicalTableName(region, tableName);
+ String storageKey = regionKey(region, canonicalTableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ String itemKey = buildItemKey(table, item);
+ JsonNode prev = putItemInternal(tableName, item, null, null, null, region, "NONE", false, pendingStreamEvents::add);
+ rollbackList.add(new RollbackEntry(storageKey, itemKey, prev));
+ affectedStorageKeys.add(storageKey);
+ } else if (transactItem.has("Delete")) {
+ JsonNode del = transactItem.get("Delete");
+ String tableName = del.path("TableName").asText();
+ JsonNode key = del.get("Key");
+ String canonicalTableName = canonicalTableName(region, tableName);
+ String storageKey = regionKey(region, canonicalTableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ String itemKey = buildItemKey(table, key, true);
+ JsonNode prev = deleteItemInternal(tableName, key, null, null, null, region, "NONE", false, pendingStreamEvents::add);
+ rollbackList.add(new RollbackEntry(storageKey, itemKey, prev));
+ affectedStorageKeys.add(storageKey);
+ } else if (transactItem.has("Update")) {
+ JsonNode upd = transactItem.get("Update");
+ String tableName = upd.path("TableName").asText();
+ JsonNode key = upd.get("Key");
+ String canonicalTableName = canonicalTableName(region, tableName);
+ String storageKey = regionKey(region, canonicalTableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ String itemKey = buildItemKey(table, key, true);
+ String updateExpression = upd.has("UpdateExpression") ? upd.get("UpdateExpression").asText() : null;
+ JsonNode exprAttrNames = upd.has("ExpressionAttributeNames") ? upd.get("ExpressionAttributeNames") : null;
+ JsonNode exprAttrValues = upd.has("ExpressionAttributeValues") ? upd.get("ExpressionAttributeValues") : null;
+ UpdateResult res = updateItemInternal(tableName, key, null, updateExpression, exprAttrNames, exprAttrValues,
+ "NONE", null, region, "NONE", false, pendingStreamEvents::add);
+ rollbackList.add(new RollbackEntry(storageKey, itemKey, res.oldItem()));
+ affectedStorageKeys.add(storageKey);
+ }
+ // ConditionCheck-only items are handled in the first pass only
}
- // ConditionCheck-only items are handled in the first pass only
+ for (String storageKey : affectedStorageKeys) {
+ persistItems(storageKey);
+ }
+ transactionSucceeded = true;
+ } finally {
+ if (!transactionSucceeded) {
+ for (int i = rollbackList.size() - 1; i >= 0; i--) {
+ RollbackEntry r = rollbackList.get(i);
+ var tableItems = itemsByTable.get(scopedItemsKey(r.storageKey()));
+ if (tableItems != null) {
+ if (r.previousValue() == null) {
+ tableItems.remove(r.itemKey());
+ } else {
+ tableItems.put(r.itemKey(), r.previousValue());
+ }
+ }
+ }
+ }
+ }
+
+ for (Runnable streamEvent : pendingStreamEvents) {
+ streamEvent.run();
}
} finally {
for (int i = acquired.size() - 1; i >= 0; i--) {
@@ -1280,6 +1460,79 @@ public class DynamoDbService implements ResourceProvider {
}
}
+ private void validateKeyNotModified(TableDefinition table, JsonNode key, JsonNode item) {
+ String pkName = table.getPartitionKeyName();
+ JsonNode origPk = key.get(pkName);
+ JsonNode newPk = item.get(pkName);
+ if (origPk != null && newPk != null && !origPk.equals(newPk)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Cannot update attribute " + pkName
+ + ". This attribute is part of the key", 400);
+ }
+ String skName = table.getSortKeyName();
+ if (skName != null) {
+ JsonNode origSk = key.get(skName);
+ JsonNode newSk = item.get(skName);
+ if (origSk != null && newSk != null && !origSk.equals(newSk)) {
+ throw new AwsException("ValidationException",
+ "One or more parameter values were invalid: Cannot update attribute " + skName
+ + ". This attribute is part of the key", 400);
+ }
+ }
+ }
+
+ private void validateTransactItem(JsonNode transactItem, String region) {
+ if (transactItem.has("Put")) {
+ JsonNode put = transactItem.get("Put");
+ String tableName = canonicalTableName(region, put.path("TableName").asText());
+ String storageKey = regionKey(region, tableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(tableName));
+ JsonNode item = put.get("Item");
+ if (item == null) {
+ throw new AwsException("ValidationException", "Item is required for Put", 400);
+ }
+ JsonNode normalizedItem = DynamoDbNumberUtils.normalizeNumbersInItem(item);
+ DynamoDbItemSize.validateSize(normalizedItem);
+ buildItemKey(table, normalizedItem);
+ validateIndexKeyTypes(table, normalizedItem, false);
+ } else if (transactItem.has("Delete")) {
+ JsonNode del = transactItem.get("Delete");
+ String tableName = canonicalTableName(region, del.path("TableName").asText());
+ String storageKey = regionKey(region, tableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(tableName));
+ JsonNode key = del.get("Key");
+ if (key == null) {
+ throw new AwsException("ValidationException", "Key is required for Delete", 400);
+ }
+ buildItemKey(table, key, true);
+ } else if (transactItem.has("Update")) {
+ JsonNode upd = transactItem.get("Update");
+ String tableName = canonicalTableName(region, upd.path("TableName").asText());
+ String storageKey = regionKey(region, tableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(tableName));
+ JsonNode key = upd.get("Key");
+ if (key == null) {
+ throw new AwsException("ValidationException", "Key is required for Update", 400);
+ }
+ String itemKey = buildItemKey(table, key, true);
+ var items = itemsByTable.get(scopedItemsKey(storageKey));
+ JsonNode existing = items != null ? items.get(itemKey) : null;
+ ObjectNode item = existing != null ? existing.deepCopy() : key.deepCopy();
+
+ String updateExpression = upd.has("UpdateExpression") ? upd.get("UpdateExpression").asText() : null;
+ JsonNode exprAttrNames = upd.has("ExpressionAttributeNames") ? upd.get("ExpressionAttributeNames") : null;
+ JsonNode exprAttrValues = upd.has("ExpressionAttributeValues") ? upd.get("ExpressionAttributeValues") : null;
+ if (updateExpression != null) {
+ applyUpdateExpression(item, updateExpression, exprAttrNames, exprAttrValues);
+ }
+ validateKeyNotModified(table, key, item);
+ validateIndexKeyTypes(table, item, true);
+ }
+ }
+
public List<JsonNode> transactGetItems(List<JsonNode> transactItems, String region) {
List<JsonNode> results = new ArrayList<>();
List<TransactionCanceledException.CancellationReason> cancelReasons = new ArrayList<>();
@@ -1355,7 +1608,7 @@ public class DynamoDbService implements ResourceProvider {
Set<String> knownAttrs = table.getAttributeDefinitions().stream()
.map(AttributeDefinition::getAttributeName)
- .collect(java.util.stream.Collectors.toSet());
+ .collect(Collectors.toSet());
if (newAttrDefs != null) newAttrDefs.forEach(ad -> knownAttrs.add(ad.getAttributeName()));
for (GlobalSecondaryIndex newGsi : gsiCreates) {
for (KeySchemaElement k : newGsi.getKeySchema()) {
@@ -1808,10 +2061,10 @@ public class DynamoDbService implements ResourceProvider {
"Invalid UpdateExpression: Incorrect operand type for operator or function", 400);
}
try {
- java.math.BigDecimal left = new java.math.BigDecimal(leftVal.get("N").asText());
- java.math.BigDecimal right = new java.math.BigDecimal(rightVal.get("N").asText());
- java.math.BigDecimal result = (operator == '+') ? left.add(right) : left.subtract(right);
- ObjectNode numNode = com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ BigDecimal left = new BigDecimal(leftVal.get("N").asText());
+ BigDecimal right = new BigDecimal(rightVal.get("N").asText());
+ BigDecimal result = (operator == '+') ? left.add(right) : left.subtract(right);
+ ObjectNode numNode = JsonNodeFactory.instance.objectNode();
numNode.put("N", result.toPlainString());
setValueAtPath(item, attrPath, numNode, exprAttrNames);
} catch (NumberFormatException e) {
@@ -1860,12 +2113,12 @@ public class DynamoDbService implements ResourceProvider {
throw new AwsException("ValidationException",
"An operand in the update expression has an incorrect data type", 400);
}
- com.fasterxml.jackson.databind.node.ArrayNode merged =
- com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.arrayNode();
+ ArrayNode merged =
+ JsonNodeFactory.instance.arrayNode();
list1.get("L").forEach(merged::add);
list2.get("L").forEach(merged::add);
- com.fasterxml.jackson.databind.node.ObjectNode result =
- com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ ObjectNode result =
+ JsonNodeFactory.instance.objectNode();
result.set("L", merged);
item.set(attrName, result);
}
@@ -2016,7 +2269,7 @@ public class DynamoDbService implements ResourceProvider {
* - For sets (SS, NS, BS): adds elements to the existing set, or creates the set if it doesn't exist
*/
private JsonNode applyAddOperation(JsonNode existingValue, JsonNode addValue) {
- ObjectNode result = com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ ObjectNode result = JsonNodeFactory.instance.objectNode();
// Handle number addition
if (addValue.has("N")) {
@@ -2028,8 +2281,8 @@ public class DynamoDbService implements ResourceProvider {
// Add the numbers
String existingNumStr = existingValue.get("N").asText();
try {
- java.math.BigDecimal existingNum = new java.math.BigDecimal(existingNumStr);
- java.math.BigDecimal addNum = new java.math.BigDecimal(addNumStr);
+ BigDecimal existingNum = new BigDecimal(existingNumStr);
+ BigDecimal addNum = new BigDecimal(addNumStr);
result.put("N", existingNum.add(addNum).toPlainString());
return result;
} catch (NumberFormatException e) {
@@ -2043,7 +2296,7 @@ public class DynamoDbService implements ResourceProvider {
if (existingValue == null || !existingValue.has("SS")) {
return addValue;
}
- java.util.Set<String> combined = new java.util.LinkedHashSet<>();
+ Set<String> combined = new LinkedHashSet<>();
existingValue.get("SS").forEach(n -> combined.add(n.asText()));
addValue.get("SS").forEach(n -> combined.add(n.asText()));
var arrayNode = result.putArray("SS");
@@ -2056,7 +2309,7 @@ public class DynamoDbService implements ResourceProvider {
if (existingValue == null || !existingValue.has("NS")) {
return addValue;
}
- java.util.Set<String> combined = new java.util.LinkedHashSet<>();
+ Set<String> combined = new LinkedHashSet<>();
existingValue.get("NS").forEach(n -> combined.add(n.asText()));
addValue.get("NS").forEach(n -> combined.add(n.asText()));
var arrayNode = result.putArray("NS");
@@ -2069,7 +2322,7 @@ public class DynamoDbService implements ResourceProvider {
if (existingValue == null || !existingValue.has("BS")) {
return addValue;
}
- java.util.Set<String> combined = new java.util.LinkedHashSet<>();
+ Set<String> combined = new LinkedHashSet<>();
existingValue.get("BS").forEach(n -> combined.add(n.asText()));
addValue.get("BS").forEach(n -> combined.add(n.asText()));
var arrayNode = result.putArray("BS");
@@ -2134,12 +2387,12 @@ public class DynamoDbService implements ResourceProvider {
* Returns the existing value unchanged if types don't match or the value isn't a set.
*/
private JsonNode applyDeleteOperation(JsonNode existingValue, JsonNode deleteValue) {
- ObjectNode result = com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ ObjectNode result = JsonNodeFactory.instance.objectNode();
if (deleteValue.has("SS") && existingValue.has("SS")) {
- java.util.Set<String> toRemove = new java.util.LinkedHashSet<>();
+ Set<String> toRemove = new LinkedHashSet<>();
deleteValue.get("SS").forEach(n -> toRemove.add(n.asText()));
- java.util.List<String> remaining = new java.util.ArrayList<>();
+ List<String> remaining = new ArrayList<>();
existingValue.get("SS").forEach(n -> {
if (!toRemove.contains(n.asText())) remaining.add(n.asText());
});
@@ -2150,9 +2403,9 @@ public class DynamoDbService implements ResourceProvider {
}
if (deleteValue.has("NS") && existingValue.has("NS")) {
- java.util.Set<String> toRemove = new java.util.LinkedHashSet<>();
+ Set<String> toRemove = new LinkedHashSet<>();
deleteValue.get("NS").forEach(n -> toRemove.add(n.asText()));
- java.util.List<String> remaining = new java.util.ArrayList<>();
+ List<String> remaining = new ArrayList<>();
existingValue.get("NS").forEach(n -> {
if (!toRemove.contains(n.asText())) remaining.add(n.asText());
});
@@ -2163,9 +2416,9 @@ public class DynamoDbService implements ResourceProvider {
}
if (deleteValue.has("BS") && existingValue.has("BS")) {
- java.util.Set<String> toRemove = new java.util.LinkedHashSet<>();
+ Set<String> toRemove = new LinkedHashSet<>();
deleteValue.get("BS").forEach(n -> toRemove.add(n.asText()));
- java.util.List<String> remaining = new java.util.ArrayList<>();
+ List<String> remaining = new ArrayList<>();
existingValue.get("BS").forEach(n -> {
if (!toRemove.contains(n.asText())) remaining.add(n.asText());
});
@@ -2217,9 +2470,9 @@ public class DynamoDbService implements ResourceProvider {
}
// Pads arr with NULL elements up to idx-1, then sets/appends value at idx.
- private void padAndSet(com.fasterxml.jackson.databind.node.ArrayNode arr, int idx, JsonNode value) {
- com.fasterxml.jackson.databind.node.ObjectNode nullNode =
- com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ private void padAndSet(ArrayNode arr, int idx, JsonNode value) {
+ ObjectNode nullNode =
+ JsonNodeFactory.instance.objectNode();
nullNode.put("NULL", true);
while (arr.size() < idx) arr.add(nullNode.deepCopy());
if (idx < arr.size()) arr.set(idx, value);
@@ -2261,7 +2514,7 @@ public class DynamoDbService implements ResourceProvider {
} else if (nextTok instanceof Integer finalIdx) {
if (child == null || !child.has("L")) throw new AwsException("ValidationException",
"The document path provided in the update expression is invalid for update", 400);
- padAndSet((com.fasterxml.jackson.databind.node.ArrayNode) child.get("L"), finalIdx, value);
+ padAndSet((ArrayNode) child.get("L"), finalIdx, value);
}
return;
}
@@ -2284,7 +2537,7 @@ public class DynamoDbService implements ResourceProvider {
container = child.get("L");
}
} else if (tok instanceof Integer listIdx) {
- if (!(container instanceof com.fasterxml.jackson.databind.node.ArrayNode arr)) return;
+ if (!(container instanceof ArrayNode arr)) return;
if (listIdx >= arr.size()) throw new AwsException("ValidationException",
"The document path provided in the update expression is invalid for update", 400);
JsonNode element = arr.get(listIdx);
@@ -2296,7 +2549,7 @@ public class DynamoDbService implements ResourceProvider {
} else if (nextTok instanceof Integer finalIdx) {
if (!element.has("L")) throw new AwsException("ValidationException",
"The document path provided in the update expression is invalid for update", 400);
- padAndSet((com.fasterxml.jackson.databind.node.ArrayNode) element.get("L"), finalIdx, value);
+ padAndSet((ArrayNode) element.get("L"), finalIdx, value);
}
return;
}
@@ -2379,8 +2632,8 @@ public class DynamoDbService implements ResourceProvider {
((ObjectNode) child.get("M")).remove(finalAttr);
} else if (nextTok instanceof Integer finalIdx) {
if (child == null || !child.has("L")) return;
- com.fasterxml.jackson.databind.node.ArrayNode lArr =
- (com.fasterxml.jackson.databind.node.ArrayNode) child.get("L");
+ ArrayNode lArr =
+ (ArrayNode) child.get("L");
if (finalIdx < lArr.size()) lArr.remove(finalIdx);
}
return;
@@ -2393,7 +2646,7 @@ public class DynamoDbService implements ResourceProvider {
container = child.get("L");
}
} else if (tok instanceof Integer listIdx) {
- if (!(container instanceof com.fasterxml.jackson.databind.node.ArrayNode arr)) return;
+ if (!(container instanceof ArrayNode arr)) return;
if (listIdx >= arr.size()) return;
JsonNode element = arr.get(listIdx);
if (last) {
@@ -2402,8 +2655,8 @@ public class DynamoDbService implements ResourceProvider {
((ObjectNode) element.get("M")).remove(finalAttr);
} else if (nextTok instanceof Integer finalIdx) {
if (!element.has("L")) return;
- com.fasterxml.jackson.databind.node.ArrayNode lArr =
- (com.fasterxml.jackson.databind.node.ArrayNode) element.get("L");
+ ArrayNode lArr =
+ (ArrayNode) element.get("L");
if (finalIdx < lArr.size()) lArr.remove(finalIdx);
}
return;
@@ -2762,8 +3015,8 @@ public class DynamoDbService implements ResourceProvider {
// loses composite key identity. See floci-io/floci#1675.
JsonNode buildKeyNode(TableDefinition table, JsonNode item,
String pkName, List<String> skNames, boolean isIndexQuery) {
- com.fasterxml.jackson.databind.node.ObjectNode keyNode =
- com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
+ ObjectNode keyNode =
+ JsonNodeFactory.instance.objectNode();
JsonNode pkAttr = item.get(pkName);
if (pkAttr != null) {
keyNode.set(pkName, pkAttr);
@@ -3102,7 +3355,7 @@ public class DynamoDbService implements ResourceProvider {
String dataKey, long itemCount, long billedSize,
String md5, String etag, String manifestSummaryKey) {
try {
- com.fasterxml.jackson.databind.node.ObjectNode root = objectMapper.createObjectNode();
+ ObjectNode root = objectMapper.createObjectNode();
root.put("version", "2020-06-30");
root.put("exportArn", desc.getExportArn());
root.put("startTime", Instant.ofEpochSecond(desc.getStartTime()).toString());
@@ -3121,8 +3374,8 @@ public class DynamoDbService implements ResourceProvider {
root.put("billedSizeBytes", billedSize);
root.put("itemCount", itemCount);
- com.fasterxml.jackson.databind.node.ArrayNode outputFiles = root.putArray("outputFiles");
- com.fasterxml.jackson.databind.node.ObjectNode fileEntry = outputFiles.addObject();
+ ArrayNode outputFiles = root.putArray("outputFiles");
+ ObjectNode fileEntry = outputFiles.addObject();
fileEntry.put("itemCount", itemCount);
fileEntry.put("md5Checksum", md5);
fileEntry.put("etag", etag);
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index da201d3776..7c14d26b1d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -1,7 +1,9 @@
package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
@@ -14,10 +16,20 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
class DynamoDbServiceTest {
@@ -381,7 +393,7 @@ class DynamoDbServiceTest {
request.set("Keys", mapper.createArrayNode().add(item("userId", "user-1")));
String usersArn = tableArn("us-east-1", "Users");
- DynamoDbService.BatchGetResult result = service.batchGetItem(java.util.Map.of(usersArn, request), "us-east-1");
+ DynamoDbService.BatchGetResult result = service.batchGetItem(Map.of(usersArn, request), "us-east-1");
assertTrue(result.responses().containsKey(usersArn));
assertEquals("Alice", result.responses().get(usersArn).getFirst().get("name").get("S").asText());
@@ -1603,7 +1615,7 @@ class DynamoDbServiceTest {
assertEquals(2, ssArray.size(), "tags should have 2 elements");
// Verify values from the SS array
- java.util.Set<String> tagValues = new java.util.HashSet<>();
+ Set<String> tagValues = new HashSet<>();
ssArray.forEach(node -> tagValues.add(node.asText()));
assertEquals(2, tagValues.size());
assertTrue(tagValues.containsAll(Arrays.asList("a", "b")));
@@ -1912,9 +1924,9 @@ class DynamoDbServiceTest {
"ADD #meta.#tags :tags", names, values, "ALL_NEW", region);
JsonNode stored = service.getItem("Users", userIdKey("nested-add-set"), region);
- java.util.Set<String> tags = new java.util.HashSet<>();
+ Set<String> tags = new HashSet<>();
stored.path("metadata").path("M").path("tags").path("SS").forEach(value -> tags.add(value.asText()));
- assertEquals(java.util.Set.of("alpha", "beta", "gamma"), tags);
+ assertEquals(Set.of("alpha", "beta", "gamma"), tags);
assertFalse(stored.has("#meta.#tags"), "ADD must not create a phantom top-level attribute");
}
@@ -3275,4 +3287,234 @@ class DynamoDbServiceTest {
JsonNode stored = service.getItem("Users", item("userId", "u1"), "us-east-1");
assertEquals("2", stored.get("counter").get("N").asText());
}
+
+ @Test
+ void batchWriteItem_flushesOncePerTable_notPerItem() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ List<JsonNode> writeRequests = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ ObjectNode item = mapper.createObjectNode();
+ item.set("userId", attributeValue("S", "u" + i));
+ ObjectNode putReq = mapper.createObjectNode();
+ putReq.set("Item", item);
+ ObjectNode req = mapper.createObjectNode();
+ req.set("PutRequest", putReq);
+ writeRequests.add(req);
+ }
+
+ serviceWithMock.batchWriteItem(Map.of("Users", writeRequests), "us-east-1");
+
+ // Verify that itemStore.put was called exactly once for the table, not 5 times
+ verify(mockItemStore, times(1))
+ .put(eq("us-east-1::Users"), any());
+ }
+
+ @Test
+ void transactWriteItems_flushesOncePerTable_notPerItem() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ List<JsonNode> transactItems = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ ObjectNode item = mapper.createObjectNode();
+ item.set("userId", attributeValue("S", "tx-u" + i));
+ ObjectNode put = mapper.createObjectNode();
+ put.put("TableName", "Users");
+ put.set("Item", item);
+ ObjectNode txItem = mapper.createObjectNode();
+ txItem.set("Put", put);
+ transactItems.add(txItem);
+ }
+
+ serviceWithMock.transactWriteItems(transactItems, "us-east-1");
+
+ verify(mockItemStore, times(1))
+ .put(eq("us-east-1::Users"), any());
+ }
+
+ @Test
+ void batchWriteItem_whenLaterItemFailsValidation_noWritesAppliedAndNoPersistence() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ // 1st item valid
+ ObjectNode item1 = mapper.createObjectNode();
+ item1.set("userId", attributeValue("S", "u1"));
+ ObjectNode putReq1 = mapper.createObjectNode();
+ putReq1.set("Item", item1);
+ ObjectNode req1 = mapper.createObjectNode();
+ req1.set("PutRequest", putReq1);
+
+ // 2nd item invalid (missing partition key "userId")
+ ObjectNode item2 = mapper.createObjectNode();
+ item2.set("otherAttr", attributeValue("S", "val"));
+ ObjectNode putReq2 = mapper.createObjectNode();
+ putReq2.set("Item", item2);
+ ObjectNode req2 = mapper.createObjectNode();
+ req2.set("PutRequest", putReq2);
+
+ assertThrows(AwsException.class, () ->
+ serviceWithMock.batchWriteItem(Map.of("Users", List.of(req1, req2)), "us-east-1"));
+
+ // Verify that itemStore.put was never called and first item was not saved in memory
+ verify(mockItemStore, never())
+ .put(any(), any());
+ assertNull(serviceWithMock.getItem("Users", item("userId", "u1"), "us-east-1"));
+ }
+
+ @Test
+ void transactWriteItems_whenLaterItemFailsValidation_noWritesAppliedAndNoPersistence() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ // 1st item: valid Put
+ ObjectNode item1 = mapper.createObjectNode();
+ item1.set("userId", attributeValue("S", "tx-u1"));
+ ObjectNode put = mapper.createObjectNode();
+ put.put("TableName", "Users");
+ put.set("Item", item1);
+ ObjectNode txItem1 = mapper.createObjectNode();
+ txItem1.set("Put", put);
+
+ // 2nd item: invalid Update that attempts to modify key attribute "userId"
+ ObjectNode key2 = mapper.createObjectNode();
+ key2.set("userId", attributeValue("S", "tx-u2"));
+ ObjectNode upd = mapper.createObjectNode();
+ upd.put("TableName", "Users");
+ upd.set("Key", key2);
+ upd.put("UpdateExpression", "SET userId = :newId");
+ ObjectNode exprValues = mapper.createObjectNode();
+ exprValues.set(":newId", attributeValue("S", "tx-u2-modified"));
+ upd.set("ExpressionAttributeValues", exprValues);
+ ObjectNode txItem2 = mapper.createObjectNode();
+ txItem2.set("Update", upd);
+
+ assertThrows(AwsException.class, () ->
+ serviceWithMock.transactWriteItems(List.of(txItem1, txItem2), "us-east-1"));
+
+ // Verify that itemStore.put was never called and first item was not retained in memory
+ verify(mockItemStore, never())
+ .put(any(), any());
+ assertNull(serviceWithMock.getItem("Users", item("userId", "tx-u1"), "us-east-1"));
+ }
+
+ @Test
+ void transactWriteItems_whenMultipleOperationsOnSameItem_failsWithValidationException() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ // 1st operation on userId "u1": Put
+ ObjectNode item1 = mapper.createObjectNode();
+ item1.set("userId", attributeValue("S", "u1"));
+ item1.set("name", attributeValue("S", "Initial"));
+ ObjectNode put = mapper.createObjectNode();
+ put.put("TableName", "Users");
+ put.set("Item", item1);
+ ObjectNode txItem1 = mapper.createObjectNode();
+ txItem1.set("Put", put);
+
+ // 2nd operation on SAME userId "u1": Update
+ ObjectNode key2 = mapper.createObjectNode();
+ key2.set("userId", attributeValue("S", "u1"));
+ ObjectNode upd = mapper.createObjectNode();
+ upd.put("TableName", "Users");
+ upd.set("Key", key2);
+ upd.put("UpdateExpression", "SET #n = :val");
+ ObjectNode exprNames = mapper.createObjectNode();
+ exprNames.put("#n", "name");
+ upd.set("ExpressionAttributeNames", exprNames);
+ ObjectNode exprValues = mapper.createObjectNode();
+ exprValues.set(":val", attributeValue("S", "Updated"));
+ upd.set("ExpressionAttributeValues", exprValues);
+ ObjectNode txItem2 = mapper.createObjectNode();
+ txItem2.set("Update", upd);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ serviceWithMock.transactWriteItems(List.of(txItem1, txItem2), "us-east-1"));
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("Transaction request cannot include multiple operations on one item"));
+
+ verify(mockItemStore, never())
+ .put(any(), any());
+ assertNull(serviceWithMock.getItem("Users", item("userId", "u1"), "us-east-1"));
+ }
+
+ @Test
+ void batchWriteItem_whenDuplicateKeysInBatch_failsWithValidationException() {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> mockItemStore = mock(StorageBackend.class);
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ DynamoDbService serviceWithMock = new DynamoDbService(
+ tableStore, mockItemStore, new RegionResolver("us-east-1", "000000000000"));
+
+ serviceWithMock.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+
+ ObjectNode item1 = mapper.createObjectNode();
+ item1.set("userId", attributeValue("S", "u1"));
+ ObjectNode putReq1 = mapper.createObjectNode();
+ putReq1.set("Item", item1);
+ ObjectNode req1 = mapper.createObjectNode();
+ req1.set("PutRequest", putReq1);
+
+ ObjectNode item2 = mapper.createObjectNode();
+ item2.set("userId", attributeValue("S", "u1"));
+ ObjectNode putReq2 = mapper.createObjectNode();
+ putReq2.set("Item", item2);
+ ObjectNode req2 = mapper.createObjectNode();
+ req2.set("PutRequest", putReq2);
+
+ AwsException ex = assertThrows(AwsException.class, () ->
+ serviceWithMock.batchWriteItem(Map.of("Users", List.of(req1, req2)), "us-east-1"));
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("Provided list of item keys contains duplicates"));
+
+ verify(mockItemStore, never())
+ .put(any(), any());
+ assertNull(serviceWithMock.getItem("Users", item("userId", "u1"), "us-east-1"));
+ }
}