Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): add missing ConsumedCapacity to transaction responses (#4151)

TATSUNO “Taz” Yasuhiro5d9986aa6cf4
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index 8127b2cada..d75a17d47e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -1671,22 +1671,7 @@ public class DynamoDbJsonHandler {
}
private Response handleTransactWriteItems(JsonNode request, String region) {
- JsonNode transactItemsNode = request.path("TransactItems");
- if (transactItemsNode.isMissingNode() || transactItemsNode.isNull() || !transactItemsNode.isArray()) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value null at 'transactItems' failed to satisfy constraint: "
- + "Member must have length greater than or equal to 1", 400);
- }
- if (transactItemsNode.isEmpty()) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value '[]' at 'transactItems' failed to satisfy constraint: "
- + "Member must have length greater than or equal to 1", 400);
- }
- if (transactItemsNode.size() > 100) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value '" + transactItemsNode + "' at 'transactItems' failed to satisfy constraint: "
- + "Member must have length less than or equal to 100", 400);
- }
+ JsonNode transactItemsNode = requireValidTransactRequest(request);
for (JsonNode txItem : transactItemsNode) {
for (JsonNode op : txItem) {
@@ -1755,13 +1740,52 @@ public class DynamoDbJsonHandler {
: null;
try {
- dynamoDbService.transactWriteItems(transactItems, region, clientRequestToken, request);
- return Response.ok(objectMapper.createObjectNode()).build();
+ DynamoDbService.TransactWriteResult result =
+ dynamoDbService.transactWriteItems(transactItems, region, clientRequestToken, request);
+ ObjectNode response = objectMapper.createObjectNode();
+ addTransactWriteConsumedCapacity(response, request, result);
+ return Response.ok(response).build();
} catch (TransactionCanceledException e) {
return transactWriteCanceled(e);
}
}
+ private static JsonNode requireValidTransactRequest(JsonNode request) {
+ JsonNode transactItems = request.path("TransactItems");
+ List<String> validationErrors = new ArrayList<>();
+ if (transactItems.isMissingNode() || transactItems.isNull() || !transactItems.isArray()) {
+ validationErrors.add("Value null at 'transactItems' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 1");
+ } else if (transactItems.isEmpty()) {
+ validationErrors.add("Value '[]' at 'transactItems' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 1");
+ } else if (transactItems.size() > 100) {
+ validationErrors.add("Value '" + transactItems + "' at 'transactItems' failed to satisfy constraint: "
+ + "Member must have length less than or equal to 100");
+ }
+ addReturnConsumedCapacityError(request, validationErrors);
+ throwValidationErrors(validationErrors);
+ return transactItems;
+ }
+
+ private static void addReturnConsumedCapacityError(JsonNode request, List<String> validationErrors) {
+ String returnCC = request.has("ReturnConsumedCapacity") ? request.get("ReturnConsumedCapacity").asText() : null;
+ if (returnCC != null && !VALID_RETURN_CONSUMED_CAPACITY.contains(returnCC)) {
+ validationErrors.add("Value '" + returnCC + "' at 'returnConsumedCapacity' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]");
+ }
+ }
+
+ private static void throwValidationErrors(List<String> validationErrors) {
+ if (validationErrors.isEmpty()) {
+ return;
+ }
+ int n = validationErrors.size();
+ throw new AwsException("ValidationException",
+ n + " validation error" + (n > 1 ? "s" : "") + " detected: "
+ + String.join("; ", validationErrors), 400);
+ }
+
private Response transactWriteCanceled(TransactionCanceledException e) {
ObjectNode body = objectMapper.createObjectNode();
body.put("__type", "TransactionCanceledException");
@@ -1782,22 +1806,7 @@ public class DynamoDbJsonHandler {
}
private Response handleTransactGetItems(JsonNode request, String region) {
- JsonNode transactItemsNode = request.path("TransactItems");
- if (transactItemsNode.isMissingNode() || transactItemsNode.isNull() || !transactItemsNode.isArray()) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value null at 'transactItems' failed to satisfy constraint: "
- + "Member must have length greater than or equal to 1", 400);
- }
- if (transactItemsNode.isEmpty()) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value '[]' at 'transactItems' failed to satisfy constraint: "
- + "Member must have length greater than or equal to 1", 400);
- }
- if (transactItemsNode.size() > 100) {
- throw new AwsException("ValidationException",
- "1 validation error detected: Value '" + transactItemsNode + "' at 'transactItems' failed to satisfy constraint: "
- + "Member must have length less than or equal to 100", 400);
- }
+ JsonNode transactItemsNode = requireValidTransactRequest(request);
for (JsonNode txItem : transactItemsNode) {
DynamoDbExpressionSize.checkRead(txItem.path("Get").path("ProjectionExpression").textValue(),
@@ -1841,9 +1850,9 @@ public class DynamoDbJsonHandler {
List<JsonNode> transactItems = new ArrayList<>();
transactItemsNode.forEach(transactItems::add);
- List<JsonNode> results;
+ DynamoDbService.TransactGetResult result;
try {
- results = dynamoDbService.transactGetItems(transactItems, region);
+ result = dynamoDbService.transactGetItems(transactItems, region);
} catch (TransactionCanceledException e) {
ObjectNode body = objectMapper.createObjectNode();
body.put("__type", "TransactionCanceledException");
@@ -1865,7 +1874,7 @@ public class DynamoDbJsonHandler {
ObjectNode response = objectMapper.createObjectNode();
ArrayNode responsesArray = objectMapper.createArrayNode();
- for (JsonNode item : results) {
+ for (JsonNode item : result.items()) {
ObjectNode entry = objectMapper.createObjectNode();
if (item != null) {
entry.set("Item", item);
@@ -1873,6 +1882,7 @@ public class DynamoDbJsonHandler {
responsesArray.add(entry);
}
response.set("Responses", responsesArray);
+ addPerTableConsumedCapacity(response, request, result.capacity(), "ReadCapacityUnits");
return Response.ok(response).build();
}
@@ -2285,32 +2295,59 @@ public class DynamoDbJsonHandler {
writeCapacityNode(tableName, cost, "INDEXES".equals(returnCC)));
}
+ private void addTransactWriteConsumedCapacity(ObjectNode response, JsonNode request,
+ DynamoDbService.TransactWriteResult result) {
+ addPerTableConsumedCapacity(response, request, result.capacity(),
+ result.replayed() ? "ReadCapacityUnits" : "WriteCapacityUnits");
+ }
+
+ private void addPerTableConsumedCapacity(ObjectNode response, JsonNode request,
+ Map<String, DynamoDbWriteCapacity.Cost> capacityByTable,
+ String unitsField) {
+ String returnCC = request.path("ReturnConsumedCapacity").asText("NONE");
+ if ("NONE".equals(returnCC) || capacityByTable.isEmpty()) {
+ return;
+ }
+ ArrayNode entries = objectMapper.createArrayNode();
+ capacityByTable.forEach((tableName, cost) ->
+ entries.add(writeCapacityNode(tableName, cost, "INDEXES".equals(returnCC), unitsField)));
+ response.set("ConsumedCapacity", entries);
+ }
+
private ObjectNode writeCapacityNode(String tableName, DynamoDbWriteCapacity.Cost cost,
boolean withBreakdown) {
- var cc = objectMapper.createObjectNode();
+ return writeCapacityNode(tableName, cost, withBreakdown, null);
+ }
+
+ private ObjectNode writeCapacityNode(String tableName, DynamoDbWriteCapacity.Cost cost,
+ boolean withBreakdown, String unitsField) {
+ ObjectNode cc = objectMapper.createObjectNode();
cc.put("TableName", DynamoDbTableNames.resolve(tableName));
- cc.put("CapacityUnits", cost.total());
+ cc.setAll(capacityUnitsNode(cost.total(), unitsField));
if (withBreakdown) {
- var tableCap = objectMapper.createObjectNode();
- tableCap.put("CapacityUnits", cost.table());
- cc.set("Table", tableCap);
+ cc.set("Table", capacityUnitsNode(cost.table(), unitsField));
if (!cost.gsi().isEmpty()) {
- cc.set("GlobalSecondaryIndexes", capacityUnitsMap(cost.gsi()));
+ cc.set("GlobalSecondaryIndexes", capacityUnitsMap(cost.gsi(), unitsField));
}
if (!cost.lsi().isEmpty()) {
- cc.set("LocalSecondaryIndexes", capacityUnitsMap(cost.lsi()));
+ cc.set("LocalSecondaryIndexes", capacityUnitsMap(cost.lsi(), unitsField));
}
}
return cc;
}
- private ObjectNode capacityUnitsMap(Map<String, Double> unitsByIndex) {
- var node = objectMapper.createObjectNode();
- unitsByIndex.forEach((indexName, units) -> {
- var cap = objectMapper.createObjectNode();
- cap.put("CapacityUnits", units);
- node.set(indexName, cap);
- });
+ private ObjectNode capacityUnitsMap(Map<String, Double> unitsByIndex, String unitsField) {
+ ObjectNode node = objectMapper.createObjectNode();
+ unitsByIndex.forEach((indexName, units) -> node.set(indexName, capacityUnitsNode(units, unitsField)));
+ return node;
+ }
+
+ private ObjectNode capacityUnitsNode(double units, String unitsField) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("CapacityUnits", units);
+ if (unitsField != null) {
+ node.put(unitsField, units);
+ }
return node;
}
@@ -2687,6 +2724,9 @@ public class DynamoDbJsonHandler {
if (stmts.isMissingNode() || !stmts.isArray() || stmts.isEmpty()) {
throw new AwsException("ValidationException", "TransactStatements must not be empty", 400);
}
+ List<String> validationErrors = new ArrayList<>();
+ addReturnConsumedCapacityError(request, validationErrors);
+ throwValidationErrors(validationErrors);
try {
List<DynamoDbPartiQLParser.Stmt> statements = new ArrayList<>();
for (int i = 0; i < stmts.size(); i++) {
@@ -2703,7 +2743,7 @@ public class DynamoDbJsonHandler {
if (reads == statements.size()) {
return executeTransactionReads(statements.stream()
.map(DynamoDbPartiQLParser.Stmt.Select.class::cast)
- .toList(), region);
+ .toList(), request, region);
}
List<DynamoDbPartiQLHandler.TransactMember> members = new ArrayList<>();
for (int i = 0; i < statements.size(); i++) {
@@ -2711,10 +2751,11 @@ public class DynamoDbJsonHandler {
members.add(inTransactStatement(i, () -> partiQLHandler.toTransactItem(stmt, region)));
}
List<JsonNode> transactItems = cancelOnMemberReasons(members);
- dynamoDbService.transactWriteItems(transactItems, region,
+ DynamoDbService.TransactWriteResult result = dynamoDbService.transactWriteItems(transactItems, region,
request.path("ClientRequestToken").asText(null), request);
ObjectNode resp = objectMapper.createObjectNode();
resp.set("Responses", objectMapper.createArrayNode());
+ addTransactWriteConsumedCapacity(resp, request, result);
return Response.ok(resp).build();
} catch (TransactionCanceledException e) {
ObjectNode body = objectMapper.createObjectNode();
@@ -2736,7 +2777,8 @@ public class DynamoDbJsonHandler {
}
}
- private Response executeTransactionReads(List<DynamoDbPartiQLParser.Stmt.Select> selects, String region) {
+ private Response executeTransactionReads(List<DynamoDbPartiQLParser.Stmt.Select> selects, JsonNode request,
+ String region) {
List<DynamoDbPartiQLHandler.TransactMember> members = new ArrayList<>();
for (int i = 0; i < selects.size(); i++) {
DynamoDbPartiQLParser.Stmt.Select select = selects.get(i);
@@ -2744,8 +2786,9 @@ public class DynamoDbJsonHandler {
}
List<JsonNode> getItems = cancelOnMemberReasons(members);
- List<JsonNode> results = dynamoDbService.transactGetItems(getItems, region);
+ DynamoDbService.TransactGetResult result = dynamoDbService.transactGetItems(getItems, region);
requireOneReadPerItem(getItems, region);
+ List<JsonNode> results = result.items();
ArrayNode responses = objectMapper.createArrayNode();
for (int i = 0; i < results.size(); i++) {
JsonNode item = partiQLHandler.projectSelected(selects.get(i), results.get(i));
@@ -2757,6 +2800,7 @@ public class DynamoDbJsonHandler {
}
ObjectNode resp = objectMapper.createObjectNode();
resp.set("Responses", responses);
+ addPerTableConsumedCapacity(resp, request, result.capacity(), "ReadCapacityUnits");
return Response.ok(resp).build();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 563cdae1ff..a9dc06ac27 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -47,6 +47,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.HexFormat;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Iterator;
import java.util.List;
@@ -56,6 +57,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.ReentrantLock;
@@ -111,7 +114,8 @@ public class DynamoDbService implements ResourceProvider {
private static final int MAX_MULTI_ATTRIBUTE_KEY_PART_SIZE = 4;
- private record IdempotencyEntry(String requestHash, long insertedAtNanos) {}
+ private record IdempotencyEntry(String requestHash, long insertedAtNanos,
+ CompletableFuture<Map<String, DynamoDbWriteCapacity.Cost>> replayCapacity) {}
private final RegionResolver regionResolver;
private final ObjectMapper objectMapper;
private DynamoDbStreamService streamService;
@@ -1400,52 +1404,57 @@ public class DynamoDbService implements ResourceProvider {
transactWriteItems(transactItems, region, null, null);
}
- public void transactWriteItems(List<JsonNode> transactItems, String region,
- String clientRequestToken, JsonNode rawRequest) {
+ public TransactWriteResult transactWriteItems(List<JsonNode> transactItems, String region,
+ String clientRequestToken, JsonNode rawRequest) {
// Idempotency check via ClientRequestToken — AWS contract:
// * Same token + identical request body → no-op success (silently dedupe).
// * Same token + different request body → IdempotentParameterMismatchException.
// * No token, or expired token → proceed normally.
- if (clientRequestToken != null && !clientRequestToken.isEmpty() && rawRequest != null) {
- String cacheKey = regionResolver.getAccountId() + "::" + region + "::" + clientRequestToken;
- String requestHash = sha256(rawRequest.toString());
+ if (clientRequestToken == null || clientRequestToken.isEmpty() || rawRequest == null) {
+ return new TransactWriteResult(applyTransactWrite(transactItems, region).writeCapacity(), false);
+ }
+ String cacheKey = regionResolver.getAccountId() + "::" + region + "::" + clientRequestToken;
+ String requestHash = sha256(rawRequest.toString());
+ for (;;) {
long nowNanos = System.nanoTime();
-
- IdempotencyEntry existing = txIdempotency.get(cacheKey);
- if (existing != null && nowNanos - existing.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS) {
- if (existing.requestHash().equals(requestHash)) {
- LOG.debugv("transactWriteItems: idempotent replay for token={0}", clientRequestToken);
- return;
- }
+ IdempotencyEntry fresh = new IdempotencyEntry(requestHash, nowNanos, new CompletableFuture<>());
+ // compute() is used so a concurrent replay with the same body collapses onto the
+ // same entry without double-applying writes.
+ IdempotencyEntry registered = txIdempotency.compute(cacheKey, (k, v) ->
+ v != null && nowNanos - v.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS ? v : fresh);
+ if (!registered.requestHash().equals(requestHash)) {
throw new AwsException("IdempotentParameterMismatchException",
"Request parameters do not match those of an in-flight or recent transaction using the same ClientRequestToken",
400);
}
-
- // Register the token. compute() is used so a concurrent replay with the same body
- // collapses onto the same entry without double-applying writes.
- IdempotencyEntry registered = txIdempotency.compute(cacheKey, (k, v) -> {
- if (v != null && nowNanos - v.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS) {
- return v;
+ if (registered == fresh) {
+ // Best-effort eviction of stale entries.
+ txIdempotency.entrySet().removeIf(e -> nowNanos - e.getValue().insertedAtNanos() > TX_IDEMPOTENCY_TTL_NANOS);
+ try {
+ AppliedTransactWrite applied = applyTransactWrite(transactItems, region);
+ fresh.replayCapacity().complete(applied.replayCapacity());
+ return new TransactWriteResult(applied.writeCapacity(), false);
+ } catch (RuntimeException e) {
+ // AWS keeps no token for a call that fails, so a retry runs the transaction again.
+ txIdempotency.remove(cacheKey, fresh);
+ fresh.replayCapacity().completeExceptionally(e);
+ throw e;
}
- return new IdempotencyEntry(requestHash, nowNanos);
- });
- if (!registered.requestHash().equals(requestHash)) {
- throw new AwsException("IdempotentParameterMismatchException",
- "Request parameters do not match those of an in-flight or recent transaction using the same ClientRequestToken",
- 400);
}
- if (registered.insertedAtNanos() != nowNanos) {
- // Lost the race to a concurrent identical request — treat as a replay.
- LOG.debugv("transactWriteItems: concurrent identical replay for token={0}", clientRequestToken);
- return;
+ // A replay that arrives while the first call is still running waits for its result.
+ LOG.debugv("transactWriteItems: idempotent replay for token={0}", clientRequestToken);
+ try {
+ return new TransactWriteResult(registered.replayCapacity().join(), true);
+ } catch (CompletionException e) {
+ LOG.debugv("transactWriteItems: first call for token={0} failed, running it again", clientRequestToken);
}
-
- // Best-effort eviction of stale entries.
- txIdempotency.entrySet().removeIf(e -> nowNanos - e.getValue().insertedAtNanos() > TX_IDEMPOTENCY_TTL_NANOS);
}
+ }
+ private record AppliedTransactWrite(Map<String, DynamoDbWriteCapacity.Cost> writeCapacity,
+ Map<String, DynamoDbWriteCapacity.Cost> replayCapacity) {}
+ private AppliedTransactWrite applyTransactWrite(List<JsonNode> transactItems, String region) {
// Acquire every participant's item lock in a deterministic (storageKey, itemKey)
// order before evaluating conditions or applying writes. Total-ordered acquisition
// prevents deadlock across concurrent transactions; ReentrantLock lets the inner
@@ -1456,8 +1465,10 @@ public class DynamoDbService implements ResourceProvider {
// user-supplied key cannot collapse distinct transaction participants.
TreeMap<TransactParticipant, ReentrantLock> toAcquire = new TreeMap<>(PARTICIPANT_ORDER);
Set<TransactParticipant> seenParticipants = new HashSet<>();
+ List<TransactParticipant> memberParticipants = new ArrayList<>();
for (JsonNode transactItem : transactItems) {
TransactParticipant p = resolveParticipant(transactItem, region);
+ memberParticipants.add(p);
if (p == null) continue;
if (!seenParticipants.add(p)) {
throw new AwsException("ValidationException",
@@ -1514,6 +1525,7 @@ public class DynamoDbService implements ResourceProvider {
throw cancelledByMember(transactItems.size(), i, e.getMessage());
}
}
+ List<JsonNode> oldImages = stagedImages(memberParticipants, staged);
for (JsonNode transactItem : transactItems) {
if (transactItem.has("Put")) {
JsonNode put = transactItem.get("Put");
@@ -1542,6 +1554,11 @@ public class DynamoDbService implements ResourceProvider {
affectedStorageKeys.add(storageKey);
}
}
+ List<JsonNode> newImages = stagedImages(memberParticipants, staged);
+ Map<String, DynamoDbWriteCapacity.Cost> writeCapacity =
+ transactCapacity(transactItems, memberParticipants, oldImages, newImages, false);
+ Map<String, DynamoDbWriteCapacity.Cost> replayCapacity =
+ transactCapacity(transactItems, memberParticipants, oldImages, newImages, true);
// Each participant remains protected by the locks acquired above. Only participant
// entries are committed, so concurrent writes to other items are never overwritten.
@@ -1562,6 +1579,7 @@ public class DynamoDbService implements ResourceProvider {
for (Runnable streamEvent : pendingStreamEvents) {
streamEvent.run();
}
+ return new AppliedTransactWrite(writeCapacity, replayCapacity);
} finally {
for (int i = acquired.size() - 1; i >= 0; i--) {
acquired.get(i).unlock();
@@ -1569,7 +1587,41 @@ public class DynamoDbService implements ResourceProvider {
}
}
- private record TransactParticipant(String storageKey, String itemKey) {}
+ private static List<JsonNode> stagedImages(List<TransactParticipant> participants,
+ Map<String, ConcurrentSkipListMap<String, JsonNode>> staged) {
+ List<JsonNode> images = new ArrayList<>();
+ for (TransactParticipant participant : participants) {
+ images.add(participant == null ? null : staged.get(participant.storageKey()).get(participant.itemKey()));
+ }
+ return images;
+ }
+
+ private Map<String, DynamoDbWriteCapacity.Cost> transactCapacity(List<JsonNode> transactItems,
+ List<TransactParticipant> participants, List<JsonNode> oldImages, List<JsonNode> newImages,
+ boolean replay) {
+ Map<String, DynamoDbWriteCapacity.Cost> byTable = new LinkedHashMap<>();
+ for (int i = 0; i < transactItems.size(); i++) {
+ TransactParticipant participant = participants.get(i);
+ if (participant == null) {
+ continue;
+ }
+ JsonNode oldItem = oldImages.get(i);
+ JsonNode newItem = newImages.get(i);
+ DynamoDbWriteCapacity.Cost cost;
+ if (replay) {
+ cost = DynamoDbTransactCapacity.read(oldItem, newItem);
+ } else if (transactItems.get(i).has("ConditionCheck")) {
+ cost = DynamoDbTransactCapacity.conditionCheck(oldItem);
+ } else {
+ cost = DynamoDbTransactCapacity.write(
+ requireActiveTable(participant.storageKey(), participant.tableName()), oldItem, newItem);
+ }
+ byTable.merge(participant.tableName(), cost, DynamoDbWriteCapacity.Cost::plus);
+ }
+ return byTable;
+ }
+
+ private record TransactParticipant(String storageKey, String itemKey, String tableName) {}
private static final Comparator<TransactParticipant> PARTICIPANT_ORDER =
Comparator.comparing(TransactParticipant::storageKey)
@@ -1600,7 +1652,7 @@ public class DynamoDbService implements ResourceProvider {
String storageKey = regionKey(region, tableName);
var table = requireActiveTable(storageKey, tableName);
String itemKey = buildItemKey(table, keyOrItem);
- return new TransactParticipant(storageKey, itemKey);
+ return new TransactParticipant(storageKey, itemKey, tableName);
}
// AWS checks the depth of a transact Update's values as part of that member, so a too
@@ -1814,8 +1866,9 @@ public class DynamoDbService implements ResourceProvider {
}
}
- public List<JsonNode> transactGetItems(List<JsonNode> transactItems, String region) {
+ public TransactGetResult transactGetItems(List<JsonNode> transactItems, String region) {
List<JsonNode> results = new ArrayList<>();
+ Map<String, DynamoDbWriteCapacity.Cost> capacity = new LinkedHashMap<>();
List<TransactionCanceledException.CancellationReason> cancelReasons = new ArrayList<>();
boolean hasCancelled = false;
@@ -1825,7 +1878,10 @@ public class DynamoDbService implements ResourceProvider {
String tableName = get.path("TableName").asText();
JsonNode key = get.get("Key");
try {
- results.add(projectTransactGet(getItem(tableName, key, region), get));
+ JsonNode item = getItem(tableName, key, region);
+ results.add(projectTransactGet(item, get));
+ capacity.merge(canonicalTableName(region, tableName), DynamoDbTransactCapacity.read(item, null),
+ DynamoDbWriteCapacity.Cost::plus);
cancelReasons.add(new TransactionCanceledException.CancellationReason("", null));
} catch (AwsException e) {
if ("ValidationException".equals(e.getErrorCode())) {
@@ -1846,7 +1902,7 @@ public class DynamoDbService implements ResourceProvider {
throw new TransactionCanceledException(cancelReasons);
}
- return results;
+ return new TransactGetResult(results, capacity);
}
/**
@@ -3822,6 +3878,10 @@ public class DynamoDbService implements ResourceProvider {
// One path the update expression acted on, with the value there before and after.
public record TouchedPath(List<Object> tokens, JsonNode oldValue, JsonNode newValue) {}
+ public record TransactWriteResult(Map<String, DynamoDbWriteCapacity.Cost> capacity, boolean replayed) {}
+
+ public record TransactGetResult(List<JsonNode> items, Map<String, DynamoDbWriteCapacity.Cost> capacity) {}
+
// scannedBytes carries the pre-filter size of the read items: DynamoDB bills a
// Query or Scan on what it read, not on what survived the filter or projection.
public record ScanResult(List<JsonNode> items, int scannedCount, long scannedBytes, JsonNode lastEvaluatedKey) {}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
new file mode 100644
index 0000000000..b398356550
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java
@@ -0,0 +1,43 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+
+import java.util.Map;
+
+/**
+ * A transaction charges the table twice the normal rate. Each index keeps the normal write
+ * rate. A ConditionCheck is charged as a write of the item it checks. A same-token replay
+ * reports a transactional read of each member instead, sized on the images of the first call.
+ */
+final class DynamoDbTransactCapacity {
+
+ private static final int WRITE_UNIT_BYTES = 1024;
+ private static final int READ_UNIT_BYTES = 4096;
+
+ private DynamoDbTransactCapacity() {}
+
+ static DynamoDbWriteCapacity.Cost write(TableDefinition table, JsonNode oldItem, JsonNode newItem) {
+ DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(table, oldItem, newItem);
+ return new DynamoDbWriteCapacity.Cost(cost.table() * 2, cost.gsi(), cost.lsi());
+ }
+
+ static DynamoDbWriteCapacity.Cost conditionCheck(JsonNode item) {
+ return tableOnly(2 * units(item, WRITE_UNIT_BYTES));
+ }
+
+ static DynamoDbWriteCapacity.Cost read(JsonNode oldItem, JsonNode newItem) {
+ return tableOnly(2 * Math.max(units(oldItem, READ_UNIT_BYTES), units(newItem, READ_UNIT_BYTES)));
+ }
+
+ private static DynamoDbWriteCapacity.Cost tableOnly(long units) {
+ return new DynamoDbWriteCapacity.Cost(units, Map.of(), Map.of());
+ }
+
+ private static long units(JsonNode item, int unitBytes) {
+ if (item == null) {
+ return 1;
+ }
+ return Math.max(1, (DynamoDbItemSize.calculateItemSize(item) + unitBytes - 1) / unitBytes);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java
index d0724de834..e93fceb8b5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.RestAssured;
+import io.restassured.response.ValidatableResponse;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
@@ -20,7 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
* the paritysuite dynamodb-conformance tier1 suite. Reads cost half a unit per 4KB,
* doubled for a strongly consistent read, and Query and Scan are sized on what was
* read before the filter. Writes charge the table plus each index whose stored view
- * the write changes, and no operation reports a read/write split.
+ * the write changes. Only transactions report a read/write split, and they charge the
+ * table twice the normal rate.
*/
@QuarkusTest
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@@ -439,6 +441,208 @@ class DynamoDbConsumedCapacityIntegrationTest {
.when().post("/").then().statusCode(200);
}
+ @Test
+ @Order(14)
+ void transactWriteChargesTheTableTwiceAndEachIndexOnce() {
+ transactWrite("""
+ {
+ "TransactItems": [{"Put": {"TableName": "%s",
+ "Item": {"pk": {"S": "cc-tx"}, "gsiPk": {"S": "t1"}, "gsi2Pk": {"S": "t2"}}}}],
+ "ReturnConsumedCapacity": "INDEXES"
+ }
+ """.formatted(TABLE))
+ .body("ConsumedCapacity", hasSize(1))
+ .body("ConsumedCapacity[0].TableName", equalTo(TABLE))
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(4.0f))
+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(4.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue())
+ .body("ConsumedCapacity[0].Table.CapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi1.WriteCapacityUnits", equalTo(1.0f))
+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi2.WriteCapacityUnits", equalTo(1.0f));
+ }
+
+ @Test
+ @Order(15)
+ void transactConditionCheckIsChargedAsAWriteOfTheCheckedItem() {
+ putItem("""
+ {"pk": {"S": "cc-tx-checked"}, "filler": {"S": "%s"}}
+ """.formatted("x".repeat(5000)));
+ transactWrite("""
+ {
+ "TransactItems": [
+ {"Put": {"TableName": "%1$s", "Item": {"pk": {"S": "cc-tx-put"}}}},
+ {"ConditionCheck": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-checked"}},
+ "ConditionExpression": "attribute_exists(pk)"}}
+ ],
+ "ReturnConsumedCapacity": "TOTAL"
+ }
+ """.formatted(TABLE))
+ .body("ConsumedCapacity", hasSize(1))
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(12.0f))
+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(12.0f))
+ .body("ConsumedCapacity[0].Table", nullValue());
+ }
+
+ @Test
+ @Order(16)
+ void transactReplayReportsAReadSizedOnTheFirstCall() {
+ String body = """
+ {
+ "ClientRequestToken": "cc-tx-replay",
+ "TransactItems": [{"Put": {"TableName": "%s",
+ "Item": {"pk": {"S": "cc-tx-replay"}, "gsiPk": {"S": "r"}, "filler": {"S": "%s"}}}}],
+ "ReturnConsumedCapacity": "INDEXES"
+ }
+ """.formatted(TABLE, "x".repeat(1536));
+ transactWrite(body)
+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(6.0f))
+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(4.0f))
+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi1.WriteCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue());
+ putItem("""
+ {"pk": {"S": "cc-tx-replay"}, "filler": {"S": "%s"}}
+ """.formatted("x".repeat(5000)));
+ transactWrite(body)
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue())
+ .body("ConsumedCapacity[0].Table.ReadCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes", nullValue());
+ }
+
+ @Test
+ @Order(17)
+ void transactGetChargesTheWholeItemAndAMissingItem() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.TransactGetItems")
+ .contentType(CT)
+ .body("""
+ {
+ "TransactItems": [
+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-checked"}},
+ "ProjectionExpression": "pk"}},
+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-missing"}}}}
+ ],
+ "ReturnConsumedCapacity": "INDEXES"
+ }
+ """.formatted(TABLE))
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("ConsumedCapacity", hasSize(1))
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(6.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(6.0f))
+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue())
+ .body("ConsumedCapacity[0].Table.ReadCapacityUnits", equalTo(6.0f));
+ }
+
+ @Test
+ @Order(18)
+ void executeTransactionIsChargedLikeATransaction() {
+ putItem("""
+ {"pk": {"S": "cc-et"}, "data": {"S": "x"}}
+ """);
+ String update = """
+ {
+ "ClientRequestToken": "cc-et-tx",
+ "TransactStatements": [{"Statement": "UPDATE \\"%s\\" SET data = 'y' WHERE pk = 'cc-et'"}],
+ "ReturnConsumedCapacity": "INDEXES"
+ }
+ """.formatted(TABLE);
+ send("ExecuteTransaction", update)
+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue());
+ send("ExecuteTransaction", update)
+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(2.0f))
+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue());
+ send("ExecuteTransaction", """
+ {
+ "TransactStatements": [
+ {"Statement": "SELECT * FROM \\"%1$s\\" WHERE pk = 'cc-et'"},
+ {"Statement": "SELECT * FROM \\"%1$s\\" WHERE pk = 'cc-et-missing'"}
+ ],
+ "ReturnConsumedCapacity": "TOTAL"
+ }
+ """.formatted(TABLE))
+ .body("ConsumedCapacity", hasSize(1))
+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(4.0f))
+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(4.0f));
+ }
+
+ @Test
+ @Order(19)
+ void transactionsRejectAnInvalidReturnConsumedCapacityBeforeWriting() {
+ String invalid = "Value 'INVALID' at 'returnConsumedCapacity' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]";
+ rejected("TransactWriteItems", """
+ {
+ "TransactItems": [{"Put": {"TableName": "%s", "Item": {"pk": {"S": "cc-rcc"}}}}],
+ "ReturnConsumedCapacity": "INVALID"
+ }
+ """.formatted(TABLE))
+ .body("message", equalTo("1 validation error detected: " + invalid));
+ rejected("ExecuteTransaction", """
+ {
+ "TransactStatements": [{"Statement": "INSERT INTO \\"%s\\" VALUE {'pk': 'cc-rcc-et'}"}],
+ "ReturnConsumedCapacity": "INVALID"
+ }
+ """.formatted(TABLE))
+ .body("message", equalTo("1 validation error detected: " + invalid));
+ rejected("TransactGetItems", """
+ {
+ "TransactItems": [{"Get": {"TableName": "%s", "Key": {"pk": {"S": "cc-get"}}}}],
+ "ReturnConsumedCapacity": "INVALID"
+ }
+ """.formatted(TABLE))
+ .body("message", equalTo("1 validation error detected: " + invalid));
+ rejected("TransactWriteItems", """
+ {"TransactItems": [], "ReturnConsumedCapacity": "INVALID"}
+ """)
+ .body("message", equalTo("2 validation errors detected: Value '[]' at 'transactItems' failed to "
+ + "satisfy constraint: Member must have length greater than or equal to 1; " + invalid));
+
+ send("TransactGetItems", """
+ {
+ "TransactItems": [
+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-rcc"}}}},
+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-rcc-et"}}}}
+ ]
+ }
+ """.formatted(TABLE))
+ .body("Responses[0].Item", nullValue())
+ .body("Responses[1].Item", nullValue());
+ }
+
+ private static ValidatableResponse rejected(String operation, String body) {
+ return given()
+ .header("X-Amz-Target", "DynamoDB_20120810." + operation)
+ .contentType(CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", equalTo("ValidationException"));
+ }
+
+ private static ValidatableResponse transactWrite(String body) {
+ return send("TransactWriteItems", body);
+ }
+
+ private static ValidatableResponse send(String operation, String body) {
+ return given()
+ .header("X-Amz-Target", "DynamoDB_20120810." + operation)
+ .contentType(CT)
+ .body(body)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+ }
+
private static void putItem(String itemJson) {
given()
.header("X-Amz-Target", "DynamoDB_20120810.PutItem")
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index b9f9612678..5aa9255710 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -30,11 +30,18 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -3740,6 +3747,70 @@ class DynamoDbServiceTest {
assertEquals("2", stored.get("counter").get("N").asText());
}
+ @Test
+ void transactWriteItemsReplayDuringTheFirstCallWaitsForItsCapacity() throws Exception {
+ @SuppressWarnings("unchecked")
+ StorageBackend<String, Map<String, JsonNode>> itemStore = mock(StorageBackend.class);
+ DynamoDbService svc = new DynamoDbService(
+ new InMemoryStorage<>(), itemStore, new RegionResolver("us-east-1", "000000000000"));
+ svc.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1");
+ CountDownLatch committing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ committing.countDown();
+ release.await();
+ return null;
+ }).when(itemStore).put(any(), any());
+ JsonNode put = mapper.readTree("""
+ {"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}}}}
+ """);
+ JsonNode rawRequest = mapper.readTree("""
+ {"ClientRequestToken":"tok-wait","TransactItems":[{"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}}}}]}
+ """);
+
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<DynamoDbService.TransactWriteResult> first = executor.submit(() ->
+ svc.transactWriteItems(List.of(put), "us-east-1", "tok-wait", rawRequest));
+ assertTrue(committing.await(5, TimeUnit.SECONDS));
+ Future<DynamoDbService.TransactWriteResult> replay = executor.submit(() ->
+ svc.transactWriteItems(List.of(put), "us-east-1", "tok-wait", rawRequest));
+ assertThrows(TimeoutException.class, () -> replay.get(200, TimeUnit.MILLISECONDS));
+ release.countDown();
+
+ assertFalse(first.get(5, TimeUnit.SECONDS).replayed());
+ DynamoDbService.TransactWriteResult replayed = replay.get(5, TimeUnit.SECONDS);
+ assertTrue(replayed.replayed());
+ assertEquals(2.0, replayed.capacity().get("Users").table());
+ } finally {
+ release.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void transactWriteItemsCancelledCallDoesNotKeepItsToken() throws Exception {
+ createUsersTable("us-east-1");
+ JsonNode put = mapper.readTree("""
+ {"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}},"ConditionExpression":"attribute_exists(userId)"}}
+ """);
+ ObjectNode rawRequest = mapper.createObjectNode();
+ rawRequest.putArray("TransactItems").add(put);
+ rawRequest.put("ClientRequestToken", "tok-cancel");
+
+ assertThrows(TransactionCanceledException.class, () ->
+ service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest));
+ assertThrows(TransactionCanceledException.class, () ->
+ service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest));
+
+ service.putItem("Users", item("userId", "u1"), "us-east-1");
+ assertFalse(service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest).replayed());
+ assertTrue(service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest).replayed());
+ }
+
@Test
void batchWriteItem_flushesOncePerTable_notPerItem() {
@SuppressWarnings("unchecked")