Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): add missing ConsumedCapacity to transaction responses (#4151)
TATSUNO “Taz” Yasuhiro5d9986aa6cf4
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex 8127b2cada..d75a17d47e 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -1671,22 +1671,7 @@ public class DynamoDbJsonHandler {}private Response handleTransactWriteItems(JsonNode request, String region) {- JsonNode transactItemsNode = request.path("TransactItems");- if (transactItemsNode.isMissingNode() || transactItemsNode.isNull() || !transactItemsNode.isArray()) {- throw new AwsException("ValidationException",- "1 validation error detected: Value null at 'transactItems' failed to satisfy constraint: "- + "Member must have length greater than or equal to 1", 400);- }- if (transactItemsNode.isEmpty()) {- throw new AwsException("ValidationException",- "1 validation error detected: Value '[]' at 'transactItems' failed to satisfy constraint: "- + "Member must have length greater than or equal to 1", 400);- }- if (transactItemsNode.size() > 100) {- throw new AwsException("ValidationException",- "1 validation error detected: Value '" + transactItemsNode + "' at 'transactItems' failed to satisfy constraint: "- + "Member must have length less than or equal to 100", 400);- }+ JsonNode transactItemsNode = requireValidTransactRequest(request);for (JsonNode txItem : transactItemsNode) {for (JsonNode op : txItem) {@@ -1755,13 +1740,52 @@ public class DynamoDbJsonHandler {: null;try {- dynamoDbService.transactWriteItems(transactItems, region, clientRequestToken, request);- return Response.ok(objectMapper.createObjectNode()).build();+ DynamoDbService.TransactWriteResult result =+ dynamoDbService.transactWriteItems(transactItems, region, clientRequestToken, request);+ ObjectNode response = objectMapper.createObjectNode();+ addTransactWriteConsumedCapacity(response, request, result);+ return Response.ok(response).build();} catch (TransactionCanceledException e) {return transactWriteCanceled(e);}}+ private static JsonNode requireValidTransactRequest(JsonNode request) {+ JsonNode transactItems = request.path("TransactItems");+ List<String> validationErrors = new ArrayList<>();+ if (transactItems.isMissingNode() || transactItems.isNull() || !transactItems.isArray()) {+ validationErrors.add("Value null at 'transactItems' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1");+ } else if (transactItems.isEmpty()) {+ validationErrors.add("Value '[]' at 'transactItems' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1");+ } else if (transactItems.size() > 100) {+ validationErrors.add("Value '" + transactItems + "' at 'transactItems' failed to satisfy constraint: "+ + "Member must have length less than or equal to 100");+ }+ addReturnConsumedCapacityError(request, validationErrors);+ throwValidationErrors(validationErrors);+ return transactItems;+ }++ private static void addReturnConsumedCapacityError(JsonNode request, List<String> validationErrors) {+ String returnCC = request.has("ReturnConsumedCapacity") ? request.get("ReturnConsumedCapacity").asText() : null;+ if (returnCC != null && !VALID_RETURN_CONSUMED_CAPACITY.contains(returnCC)) {+ validationErrors.add("Value '" + returnCC + "' at 'returnConsumedCapacity' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]");+ }+ }++ private static void throwValidationErrors(List<String> validationErrors) {+ if (validationErrors.isEmpty()) {+ return;+ }+ int n = validationErrors.size();+ throw new AwsException("ValidationException",+ n + " validation error" + (n > 1 ? "s" : "") + " detected: "+ + String.join("; ", validationErrors), 400);+ }+private Response transactWriteCanceled(TransactionCanceledException e) {ObjectNode body = objectMapper.createObjectNode();body.put("__type", "TransactionCanceledException");@@ -1782,22 +1806,7 @@ public class DynamoDbJsonHandler {}private Response handleTransactGetItems(JsonNode request, String region) {- JsonNode transactItemsNode = request.path("TransactItems");- if (transactItemsNode.isMissingNode() || transactItemsNode.isNull() || !transactItemsNode.isArray()) {- throw new AwsException("ValidationException",- "1 validation error detected: Value null at 'transactItems' failed to satisfy constraint: "- + "Member must have length greater than or equal to 1", 400);- }- if (transactItemsNode.isEmpty()) {- throw new AwsException("ValidationException",- "1 validation error detected: Value '[]' at 'transactItems' failed to satisfy constraint: "- + "Member must have length greater than or equal to 1", 400);- }- if (transactItemsNode.size() > 100) {- throw new AwsException("ValidationException",- "1 validation error detected: Value '" + transactItemsNode + "' at 'transactItems' failed to satisfy constraint: "- + "Member must have length less than or equal to 100", 400);- }+ JsonNode transactItemsNode = requireValidTransactRequest(request);for (JsonNode txItem : transactItemsNode) {DynamoDbExpressionSize.checkRead(txItem.path("Get").path("ProjectionExpression").textValue(),@@ -1841,9 +1850,9 @@ public class DynamoDbJsonHandler {List<JsonNode> transactItems = new ArrayList<>();transactItemsNode.forEach(transactItems::add);- List<JsonNode> results;+ DynamoDbService.TransactGetResult result;try {- results = dynamoDbService.transactGetItems(transactItems, region);+ result = dynamoDbService.transactGetItems(transactItems, region);} catch (TransactionCanceledException e) {ObjectNode body = objectMapper.createObjectNode();body.put("__type", "TransactionCanceledException");@@ -1865,7 +1874,7 @@ public class DynamoDbJsonHandler {ObjectNode response = objectMapper.createObjectNode();ArrayNode responsesArray = objectMapper.createArrayNode();- for (JsonNode item : results) {+ for (JsonNode item : result.items()) {ObjectNode entry = objectMapper.createObjectNode();if (item != null) {entry.set("Item", item);@@ -1873,6 +1882,7 @@ public class DynamoDbJsonHandler {responsesArray.add(entry);}response.set("Responses", responsesArray);+ addPerTableConsumedCapacity(response, request, result.capacity(), "ReadCapacityUnits");return Response.ok(response).build();}@@ -2285,32 +2295,59 @@ public class DynamoDbJsonHandler {writeCapacityNode(tableName, cost, "INDEXES".equals(returnCC)));}+ private void addTransactWriteConsumedCapacity(ObjectNode response, JsonNode request,+ DynamoDbService.TransactWriteResult result) {+ addPerTableConsumedCapacity(response, request, result.capacity(),+ result.replayed() ? "ReadCapacityUnits" : "WriteCapacityUnits");+ }++ private void addPerTableConsumedCapacity(ObjectNode response, JsonNode request,+ Map<String, DynamoDbWriteCapacity.Cost> capacityByTable,+ String unitsField) {+ String returnCC = request.path("ReturnConsumedCapacity").asText("NONE");+ if ("NONE".equals(returnCC) || capacityByTable.isEmpty()) {+ return;+ }+ ArrayNode entries = objectMapper.createArrayNode();+ capacityByTable.forEach((tableName, cost) ->+ entries.add(writeCapacityNode(tableName, cost, "INDEXES".equals(returnCC), unitsField)));+ response.set("ConsumedCapacity", entries);+ }+private ObjectNode writeCapacityNode(String tableName, DynamoDbWriteCapacity.Cost cost,boolean withBreakdown) {- var cc = objectMapper.createObjectNode();+ return writeCapacityNode(tableName, cost, withBreakdown, null);+ }++ private ObjectNode writeCapacityNode(String tableName, DynamoDbWriteCapacity.Cost cost,+ boolean withBreakdown, String unitsField) {+ ObjectNode cc = objectMapper.createObjectNode();cc.put("TableName", DynamoDbTableNames.resolve(tableName));- cc.put("CapacityUnits", cost.total());+ cc.setAll(capacityUnitsNode(cost.total(), unitsField));if (withBreakdown) {- var tableCap = objectMapper.createObjectNode();- tableCap.put("CapacityUnits", cost.table());- cc.set("Table", tableCap);+ cc.set("Table", capacityUnitsNode(cost.table(), unitsField));if (!cost.gsi().isEmpty()) {- cc.set("GlobalSecondaryIndexes", capacityUnitsMap(cost.gsi()));+ cc.set("GlobalSecondaryIndexes", capacityUnitsMap(cost.gsi(), unitsField));}if (!cost.lsi().isEmpty()) {- cc.set("LocalSecondaryIndexes", capacityUnitsMap(cost.lsi()));+ cc.set("LocalSecondaryIndexes", capacityUnitsMap(cost.lsi(), unitsField));}}return cc;}- private ObjectNode capacityUnitsMap(Map<String, Double> unitsByIndex) {- var node = objectMapper.createObjectNode();- unitsByIndex.forEach((indexName, units) -> {- var cap = objectMapper.createObjectNode();- cap.put("CapacityUnits", units);- node.set(indexName, cap);- });+ private ObjectNode capacityUnitsMap(Map<String, Double> unitsByIndex, String unitsField) {+ ObjectNode node = objectMapper.createObjectNode();+ unitsByIndex.forEach((indexName, units) -> node.set(indexName, capacityUnitsNode(units, unitsField)));+ return node;+ }++ private ObjectNode capacityUnitsNode(double units, String unitsField) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("CapacityUnits", units);+ if (unitsField != null) {+ node.put(unitsField, units);+ }return node;}@@ -2687,6 +2724,9 @@ public class DynamoDbJsonHandler {if (stmts.isMissingNode() || !stmts.isArray() || stmts.isEmpty()) {throw new AwsException("ValidationException", "TransactStatements must not be empty", 400);}+ List<String> validationErrors = new ArrayList<>();+ addReturnConsumedCapacityError(request, validationErrors);+ throwValidationErrors(validationErrors);try {List<DynamoDbPartiQLParser.Stmt> statements = new ArrayList<>();for (int i = 0; i < stmts.size(); i++) {@@ -2703,7 +2743,7 @@ public class DynamoDbJsonHandler {if (reads == statements.size()) {return executeTransactionReads(statements.stream().map(DynamoDbPartiQLParser.Stmt.Select.class::cast)- .toList(), region);+ .toList(), request, region);}List<DynamoDbPartiQLHandler.TransactMember> members = new ArrayList<>();for (int i = 0; i < statements.size(); i++) {@@ -2711,10 +2751,11 @@ public class DynamoDbJsonHandler {members.add(inTransactStatement(i, () -> partiQLHandler.toTransactItem(stmt, region)));}List<JsonNode> transactItems = cancelOnMemberReasons(members);- dynamoDbService.transactWriteItems(transactItems, region,+ DynamoDbService.TransactWriteResult result = dynamoDbService.transactWriteItems(transactItems, region,request.path("ClientRequestToken").asText(null), request);ObjectNode resp = objectMapper.createObjectNode();resp.set("Responses", objectMapper.createArrayNode());+ addTransactWriteConsumedCapacity(resp, request, result);return Response.ok(resp).build();} catch (TransactionCanceledException e) {ObjectNode body = objectMapper.createObjectNode();@@ -2736,7 +2777,8 @@ public class DynamoDbJsonHandler {}}- private Response executeTransactionReads(List<DynamoDbPartiQLParser.Stmt.Select> selects, String region) {+ private Response executeTransactionReads(List<DynamoDbPartiQLParser.Stmt.Select> selects, JsonNode request,+ String region) {List<DynamoDbPartiQLHandler.TransactMember> members = new ArrayList<>();for (int i = 0; i < selects.size(); i++) {DynamoDbPartiQLParser.Stmt.Select select = selects.get(i);@@ -2744,8 +2786,9 @@ public class DynamoDbJsonHandler {}List<JsonNode> getItems = cancelOnMemberReasons(members);- List<JsonNode> results = dynamoDbService.transactGetItems(getItems, region);+ DynamoDbService.TransactGetResult result = dynamoDbService.transactGetItems(getItems, region);requireOneReadPerItem(getItems, region);+ List<JsonNode> results = result.items();ArrayNode responses = objectMapper.createArrayNode();for (int i = 0; i < results.size(); i++) {JsonNode item = partiQLHandler.projectSelected(selects.get(i), results.get(i));@@ -2757,6 +2800,7 @@ public class DynamoDbJsonHandler {}ObjectNode resp = objectMapper.createObjectNode();resp.set("Responses", responses);+ addPerTableConsumedCapacity(resp, request, result.capacity(), "ReadCapacityUnits");return Response.ok(resp).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 563cdae1ff..a9dc06ac27 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -47,6 +47,7 @@ import java.util.Comparator;import java.util.HashMap;import java.util.HashSet;import java.util.HexFormat;+import java.util.LinkedHashMap;import java.util.LinkedHashSet;import java.util.Iterator;import java.util.List;@@ -56,6 +57,8 @@ import java.util.Optional;import java.util.Set;import java.util.TreeMap;import java.util.UUID;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.CompletionException;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentSkipListMap;import java.util.concurrent.locks.ReentrantLock;@@ -111,7 +114,8 @@ public class DynamoDbService implements ResourceProvider {private static final int MAX_MULTI_ATTRIBUTE_KEY_PART_SIZE = 4;- private record IdempotencyEntry(String requestHash, long insertedAtNanos) {}+ private record IdempotencyEntry(String requestHash, long insertedAtNanos,+ CompletableFuture<Map<String, DynamoDbWriteCapacity.Cost>> replayCapacity) {}private final RegionResolver regionResolver;private final ObjectMapper objectMapper;private DynamoDbStreamService streamService;@@ -1400,52 +1404,57 @@ public class DynamoDbService implements ResourceProvider {transactWriteItems(transactItems, region, null, null);}- public void transactWriteItems(List<JsonNode> transactItems, String region,- String clientRequestToken, JsonNode rawRequest) {+ public TransactWriteResult transactWriteItems(List<JsonNode> transactItems, String region,+ String clientRequestToken, JsonNode rawRequest) {// Idempotency check via ClientRequestToken — AWS contract:// * Same token + identical request body → no-op success (silently dedupe).// * Same token + different request body → IdempotentParameterMismatchException.// * No token, or expired token → proceed normally.- if (clientRequestToken != null && !clientRequestToken.isEmpty() && rawRequest != null) {- String cacheKey = regionResolver.getAccountId() + "::" + region + "::" + clientRequestToken;- String requestHash = sha256(rawRequest.toString());+ if (clientRequestToken == null || clientRequestToken.isEmpty() || rawRequest == null) {+ return new TransactWriteResult(applyTransactWrite(transactItems, region).writeCapacity(), false);+ }+ String cacheKey = regionResolver.getAccountId() + "::" + region + "::" + clientRequestToken;+ String requestHash = sha256(rawRequest.toString());+ for (;;) {long nowNanos = System.nanoTime();-- IdempotencyEntry existing = txIdempotency.get(cacheKey);- if (existing != null && nowNanos - existing.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS) {- if (existing.requestHash().equals(requestHash)) {- LOG.debugv("transactWriteItems: idempotent replay for token={0}", clientRequestToken);- return;- }+ IdempotencyEntry fresh = new IdempotencyEntry(requestHash, nowNanos, new CompletableFuture<>());+ // compute() is used so a concurrent replay with the same body collapses onto the+ // same entry without double-applying writes.+ IdempotencyEntry registered = txIdempotency.compute(cacheKey, (k, v) ->+ v != null && nowNanos - v.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS ? v : fresh);+ if (!registered.requestHash().equals(requestHash)) {throw new AwsException("IdempotentParameterMismatchException","Request parameters do not match those of an in-flight or recent transaction using the same ClientRequestToken",400);}-- // Register the token. compute() is used so a concurrent replay with the same body- // collapses onto the same entry without double-applying writes.- IdempotencyEntry registered = txIdempotency.compute(cacheKey, (k, v) -> {- if (v != null && nowNanos - v.insertedAtNanos() <= TX_IDEMPOTENCY_TTL_NANOS) {- return v;+ if (registered == fresh) {+ // Best-effort eviction of stale entries.+ txIdempotency.entrySet().removeIf(e -> nowNanos - e.getValue().insertedAtNanos() > TX_IDEMPOTENCY_TTL_NANOS);+ try {+ AppliedTransactWrite applied = applyTransactWrite(transactItems, region);+ fresh.replayCapacity().complete(applied.replayCapacity());+ return new TransactWriteResult(applied.writeCapacity(), false);+ } catch (RuntimeException e) {+ // AWS keeps no token for a call that fails, so a retry runs the transaction again.+ txIdempotency.remove(cacheKey, fresh);+ fresh.replayCapacity().completeExceptionally(e);+ throw e;}- return new IdempotencyEntry(requestHash, nowNanos);- });- if (!registered.requestHash().equals(requestHash)) {- throw new AwsException("IdempotentParameterMismatchException",- "Request parameters do not match those of an in-flight or recent transaction using the same ClientRequestToken",- 400);}- if (registered.insertedAtNanos() != nowNanos) {- // Lost the race to a concurrent identical request — treat as a replay.- LOG.debugv("transactWriteItems: concurrent identical replay for token={0}", clientRequestToken);- return;+ // A replay that arrives while the first call is still running waits for its result.+ LOG.debugv("transactWriteItems: idempotent replay for token={0}", clientRequestToken);+ try {+ return new TransactWriteResult(registered.replayCapacity().join(), true);+ } catch (CompletionException e) {+ LOG.debugv("transactWriteItems: first call for token={0} failed, running it again", clientRequestToken);}-- // Best-effort eviction of stale entries.- txIdempotency.entrySet().removeIf(e -> nowNanos - e.getValue().insertedAtNanos() > TX_IDEMPOTENCY_TTL_NANOS);}+ }+ private record AppliedTransactWrite(Map<String, DynamoDbWriteCapacity.Cost> writeCapacity,+ Map<String, DynamoDbWriteCapacity.Cost> replayCapacity) {}+ private AppliedTransactWrite applyTransactWrite(List<JsonNode> transactItems, String region) {// Acquire every participant's item lock in a deterministic (storageKey, itemKey)// order before evaluating conditions or applying writes. Total-ordered acquisition// prevents deadlock across concurrent transactions; ReentrantLock lets the inner@@ -1456,8 +1465,10 @@ public class DynamoDbService implements ResourceProvider {// user-supplied key cannot collapse distinct transaction participants.TreeMap<TransactParticipant, ReentrantLock> toAcquire = new TreeMap<>(PARTICIPANT_ORDER);Set<TransactParticipant> seenParticipants = new HashSet<>();+ List<TransactParticipant> memberParticipants = new ArrayList<>();for (JsonNode transactItem : transactItems) {TransactParticipant p = resolveParticipant(transactItem, region);+ memberParticipants.add(p);if (p == null) continue;if (!seenParticipants.add(p)) {throw new AwsException("ValidationException",@@ -1514,6 +1525,7 @@ public class DynamoDbService implements ResourceProvider {throw cancelledByMember(transactItems.size(), i, e.getMessage());}}+ List<JsonNode> oldImages = stagedImages(memberParticipants, staged);for (JsonNode transactItem : transactItems) {if (transactItem.has("Put")) {JsonNode put = transactItem.get("Put");@@ -1542,6 +1554,11 @@ public class DynamoDbService implements ResourceProvider {affectedStorageKeys.add(storageKey);}}+ List<JsonNode> newImages = stagedImages(memberParticipants, staged);+ Map<String, DynamoDbWriteCapacity.Cost> writeCapacity =+ transactCapacity(transactItems, memberParticipants, oldImages, newImages, false);+ Map<String, DynamoDbWriteCapacity.Cost> replayCapacity =+ transactCapacity(transactItems, memberParticipants, oldImages, newImages, true);// Each participant remains protected by the locks acquired above. Only participant// entries are committed, so concurrent writes to other items are never overwritten.@@ -1562,6 +1579,7 @@ public class DynamoDbService implements ResourceProvider {for (Runnable streamEvent : pendingStreamEvents) {streamEvent.run();}+ return new AppliedTransactWrite(writeCapacity, replayCapacity);} finally {for (int i = acquired.size() - 1; i >= 0; i--) {acquired.get(i).unlock();@@ -1569,7 +1587,41 @@ public class DynamoDbService implements ResourceProvider {}}- private record TransactParticipant(String storageKey, String itemKey) {}+ private static List<JsonNode> stagedImages(List<TransactParticipant> participants,+ Map<String, ConcurrentSkipListMap<String, JsonNode>> staged) {+ List<JsonNode> images = new ArrayList<>();+ for (TransactParticipant participant : participants) {+ images.add(participant == null ? null : staged.get(participant.storageKey()).get(participant.itemKey()));+ }+ return images;+ }++ private Map<String, DynamoDbWriteCapacity.Cost> transactCapacity(List<JsonNode> transactItems,+ List<TransactParticipant> participants, List<JsonNode> oldImages, List<JsonNode> newImages,+ boolean replay) {+ Map<String, DynamoDbWriteCapacity.Cost> byTable = new LinkedHashMap<>();+ for (int i = 0; i < transactItems.size(); i++) {+ TransactParticipant participant = participants.get(i);+ if (participant == null) {+ continue;+ }+ JsonNode oldItem = oldImages.get(i);+ JsonNode newItem = newImages.get(i);+ DynamoDbWriteCapacity.Cost cost;+ if (replay) {+ cost = DynamoDbTransactCapacity.read(oldItem, newItem);+ } else if (transactItems.get(i).has("ConditionCheck")) {+ cost = DynamoDbTransactCapacity.conditionCheck(oldItem);+ } else {+ cost = DynamoDbTransactCapacity.write(+ requireActiveTable(participant.storageKey(), participant.tableName()), oldItem, newItem);+ }+ byTable.merge(participant.tableName(), cost, DynamoDbWriteCapacity.Cost::plus);+ }+ return byTable;+ }++ private record TransactParticipant(String storageKey, String itemKey, String tableName) {}private static final Comparator<TransactParticipant> PARTICIPANT_ORDER =Comparator.comparing(TransactParticipant::storageKey)@@ -1600,7 +1652,7 @@ public class DynamoDbService implements ResourceProvider {String storageKey = regionKey(region, tableName);var table = requireActiveTable(storageKey, tableName);String itemKey = buildItemKey(table, keyOrItem);- return new TransactParticipant(storageKey, itemKey);+ return new TransactParticipant(storageKey, itemKey, tableName);}// AWS checks the depth of a transact Update's values as part of that member, so a too@@ -1814,8 +1866,9 @@ public class DynamoDbService implements ResourceProvider {}}- public List<JsonNode> transactGetItems(List<JsonNode> transactItems, String region) {+ public TransactGetResult transactGetItems(List<JsonNode> transactItems, String region) {List<JsonNode> results = new ArrayList<>();+ Map<String, DynamoDbWriteCapacity.Cost> capacity = new LinkedHashMap<>();List<TransactionCanceledException.CancellationReason> cancelReasons = new ArrayList<>();boolean hasCancelled = false;@@ -1825,7 +1878,10 @@ public class DynamoDbService implements ResourceProvider {String tableName = get.path("TableName").asText();JsonNode key = get.get("Key");try {- results.add(projectTransactGet(getItem(tableName, key, region), get));+ JsonNode item = getItem(tableName, key, region);+ results.add(projectTransactGet(item, get));+ capacity.merge(canonicalTableName(region, tableName), DynamoDbTransactCapacity.read(item, null),+ DynamoDbWriteCapacity.Cost::plus);cancelReasons.add(new TransactionCanceledException.CancellationReason("", null));} catch (AwsException e) {if ("ValidationException".equals(e.getErrorCode())) {@@ -1846,7 +1902,7 @@ public class DynamoDbService implements ResourceProvider {throw new TransactionCanceledException(cancelReasons);}- return results;+ return new TransactGetResult(results, capacity);}/**@@ -3822,6 +3878,10 @@ public class DynamoDbService implements ResourceProvider {// One path the update expression acted on, with the value there before and after.public record TouchedPath(List<Object> tokens, JsonNode oldValue, JsonNode newValue) {}+ public record TransactWriteResult(Map<String, DynamoDbWriteCapacity.Cost> capacity, boolean replayed) {}++ public record TransactGetResult(List<JsonNode> items, Map<String, DynamoDbWriteCapacity.Cost> capacity) {}+// scannedBytes carries the pre-filter size of the read items: DynamoDB bills a// Query or Scan on what it read, not on what survived the filter or projection.public record ScanResult(List<JsonNode> items, int scannedCount, long scannedBytes, JsonNode lastEvaluatedKey) {}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.javanew file mode 100644index 0000000000..b398356550--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTransactCapacity.java@@ -0,0 +1,43 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;++import java.util.Map;++/**+ * A transaction charges the table twice the normal rate. Each index keeps the normal write+ * rate. A ConditionCheck is charged as a write of the item it checks. A same-token replay+ * reports a transactional read of each member instead, sized on the images of the first call.+ */+final class DynamoDbTransactCapacity {++ private static final int WRITE_UNIT_BYTES = 1024;+ private static final int READ_UNIT_BYTES = 4096;++ private DynamoDbTransactCapacity() {}++ static DynamoDbWriteCapacity.Cost write(TableDefinition table, JsonNode oldItem, JsonNode newItem) {+ DynamoDbWriteCapacity.Cost cost = DynamoDbWriteCapacity.forWrite(table, oldItem, newItem);+ return new DynamoDbWriteCapacity.Cost(cost.table() * 2, cost.gsi(), cost.lsi());+ }++ static DynamoDbWriteCapacity.Cost conditionCheck(JsonNode item) {+ return tableOnly(2 * units(item, WRITE_UNIT_BYTES));+ }++ static DynamoDbWriteCapacity.Cost read(JsonNode oldItem, JsonNode newItem) {+ return tableOnly(2 * Math.max(units(oldItem, READ_UNIT_BYTES), units(newItem, READ_UNIT_BYTES)));+ }++ private static DynamoDbWriteCapacity.Cost tableOnly(long units) {+ return new DynamoDbWriteCapacity.Cost(units, Map.of(), Map.of());+ }++ private static long units(JsonNode item, int unitBytes) {+ if (item == null) {+ return 1;+ }+ return Math.max(1, (DynamoDbItemSize.calculateItemSize(item) + unitBytes - 1) / unitBytes);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.javaindex d0724de834..e93fceb8b5 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConsumedCapacityIntegrationTest.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.dynamodb;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.RestAssured;+import io.restassured.response.ValidatableResponse;import org.junit.jupiter.api.AfterAll;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.MethodOrderer;@@ -20,7 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;* the paritysuite dynamodb-conformance tier1 suite. Reads cost half a unit per 4KB,* doubled for a strongly consistent read, and Query and Scan are sized on what was* read before the filter. Writes charge the table plus each index whose stored view- * the write changes, and no operation reports a read/write split.+ * the write changes. Only transactions report a read/write split, and they charge the+ * table twice the normal rate.*/@QuarkusTest@TestMethodOrder(MethodOrderer.OrderAnnotation.class)@@ -439,6 +441,208 @@ class DynamoDbConsumedCapacityIntegrationTest {.when().post("/").then().statusCode(200);}+ @Test+ @Order(14)+ void transactWriteChargesTheTableTwiceAndEachIndexOnce() {+ transactWrite("""+ {+ "TransactItems": [{"Put": {"TableName": "%s",+ "Item": {"pk": {"S": "cc-tx"}, "gsiPk": {"S": "t1"}, "gsi2Pk": {"S": "t2"}}}}],+ "ReturnConsumedCapacity": "INDEXES"+ }+ """.formatted(TABLE))+ .body("ConsumedCapacity", hasSize(1))+ .body("ConsumedCapacity[0].TableName", equalTo(TABLE))+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(4.0f))+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(4.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue())+ .body("ConsumedCapacity[0].Table.CapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi1.WriteCapacityUnits", equalTo(1.0f))+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi2.WriteCapacityUnits", equalTo(1.0f));+ }++ @Test+ @Order(15)+ void transactConditionCheckIsChargedAsAWriteOfTheCheckedItem() {+ putItem("""+ {"pk": {"S": "cc-tx-checked"}, "filler": {"S": "%s"}}+ """.formatted("x".repeat(5000)));+ transactWrite("""+ {+ "TransactItems": [+ {"Put": {"TableName": "%1$s", "Item": {"pk": {"S": "cc-tx-put"}}}},+ {"ConditionCheck": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-checked"}},+ "ConditionExpression": "attribute_exists(pk)"}}+ ],+ "ReturnConsumedCapacity": "TOTAL"+ }+ """.formatted(TABLE))+ .body("ConsumedCapacity", hasSize(1))+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(12.0f))+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(12.0f))+ .body("ConsumedCapacity[0].Table", nullValue());+ }++ @Test+ @Order(16)+ void transactReplayReportsAReadSizedOnTheFirstCall() {+ String body = """+ {+ "ClientRequestToken": "cc-tx-replay",+ "TransactItems": [{"Put": {"TableName": "%s",+ "Item": {"pk": {"S": "cc-tx-replay"}, "gsiPk": {"S": "r"}, "filler": {"S": "%s"}}}}],+ "ReturnConsumedCapacity": "INDEXES"+ }+ """.formatted(TABLE, "x".repeat(1536));+ transactWrite(body)+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(6.0f))+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(4.0f))+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes.gsi1.WriteCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue());+ putItem("""+ {"pk": {"S": "cc-tx-replay"}, "filler": {"S": "%s"}}+ """.formatted("x".repeat(5000)));+ transactWrite(body)+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue())+ .body("ConsumedCapacity[0].Table.ReadCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].GlobalSecondaryIndexes", nullValue());+ }++ @Test+ @Order(17)+ void transactGetChargesTheWholeItemAndAMissingItem() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.TransactGetItems")+ .contentType(CT)+ .body("""+ {+ "TransactItems": [+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-checked"}},+ "ProjectionExpression": "pk"}},+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-tx-missing"}}}}+ ],+ "ReturnConsumedCapacity": "INDEXES"+ }+ """.formatted(TABLE))+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("ConsumedCapacity", hasSize(1))+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(6.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(6.0f))+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue())+ .body("ConsumedCapacity[0].Table.ReadCapacityUnits", equalTo(6.0f));+ }++ @Test+ @Order(18)+ void executeTransactionIsChargedLikeATransaction() {+ putItem("""+ {"pk": {"S": "cc-et"}, "data": {"S": "x"}}+ """);+ String update = """+ {+ "ClientRequestToken": "cc-et-tx",+ "TransactStatements": [{"Statement": "UPDATE \\"%s\\" SET data = 'y' WHERE pk = 'cc-et'"}],+ "ReturnConsumedCapacity": "INDEXES"+ }+ """.formatted(TABLE);+ send("ExecuteTransaction", update)+ .body("ConsumedCapacity[0].WriteCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].Table.WriteCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", nullValue());+ send("ExecuteTransaction", update)+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(2.0f))+ .body("ConsumedCapacity[0].WriteCapacityUnits", nullValue());+ send("ExecuteTransaction", """+ {+ "TransactStatements": [+ {"Statement": "SELECT * FROM \\"%1$s\\" WHERE pk = 'cc-et'"},+ {"Statement": "SELECT * FROM \\"%1$s\\" WHERE pk = 'cc-et-missing'"}+ ],+ "ReturnConsumedCapacity": "TOTAL"+ }+ """.formatted(TABLE))+ .body("ConsumedCapacity", hasSize(1))+ .body("ConsumedCapacity[0].CapacityUnits", equalTo(4.0f))+ .body("ConsumedCapacity[0].ReadCapacityUnits", equalTo(4.0f));+ }++ @Test+ @Order(19)+ void transactionsRejectAnInvalidReturnConsumedCapacityBeforeWriting() {+ String invalid = "Value 'INVALID' at 'returnConsumedCapacity' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]";+ rejected("TransactWriteItems", """+ {+ "TransactItems": [{"Put": {"TableName": "%s", "Item": {"pk": {"S": "cc-rcc"}}}}],+ "ReturnConsumedCapacity": "INVALID"+ }+ """.formatted(TABLE))+ .body("message", equalTo("1 validation error detected: " + invalid));+ rejected("ExecuteTransaction", """+ {+ "TransactStatements": [{"Statement": "INSERT INTO \\"%s\\" VALUE {'pk': 'cc-rcc-et'}"}],+ "ReturnConsumedCapacity": "INVALID"+ }+ """.formatted(TABLE))+ .body("message", equalTo("1 validation error detected: " + invalid));+ rejected("TransactGetItems", """+ {+ "TransactItems": [{"Get": {"TableName": "%s", "Key": {"pk": {"S": "cc-get"}}}}],+ "ReturnConsumedCapacity": "INVALID"+ }+ """.formatted(TABLE))+ .body("message", equalTo("1 validation error detected: " + invalid));+ rejected("TransactWriteItems", """+ {"TransactItems": [], "ReturnConsumedCapacity": "INVALID"}+ """)+ .body("message", equalTo("2 validation errors detected: Value '[]' at 'transactItems' failed to "+ + "satisfy constraint: Member must have length greater than or equal to 1; " + invalid));++ send("TransactGetItems", """+ {+ "TransactItems": [+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-rcc"}}}},+ {"Get": {"TableName": "%1$s", "Key": {"pk": {"S": "cc-rcc-et"}}}}+ ]+ }+ """.formatted(TABLE))+ .body("Responses[0].Item", nullValue())+ .body("Responses[1].Item", nullValue());+ }++ private static ValidatableResponse rejected(String operation, String body) {+ return given()+ .header("X-Amz-Target", "DynamoDB_20120810." + operation)+ .contentType(CT)+ .body(body)+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("ValidationException"));+ }++ private static ValidatableResponse transactWrite(String body) {+ return send("TransactWriteItems", body);+ }++ private static ValidatableResponse send(String operation, String body) {+ return given()+ .header("X-Amz-Target", "DynamoDB_20120810." + operation)+ .contentType(CT)+ .body(body)+ .when()+ .post("/")+ .then()+ .statusCode(200);+ }+private static void putItem(String itemJson) {given().header("X-Amz-Target", "DynamoDB_20120810.PutItem")diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex b9f9612678..5aa9255710 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -30,11 +30,18 @@ import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.zip.GZIPOutputStream;import static org.junit.jupiter.api.Assertions.*;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.times;@@ -3740,6 +3747,70 @@ class DynamoDbServiceTest {assertEquals("2", stored.get("counter").get("N").asText());}+ @Test+ void transactWriteItemsReplayDuringTheFirstCallWaitsForItsCapacity() throws Exception {+ @SuppressWarnings("unchecked")+ StorageBackend<String, Map<String, JsonNode>> itemStore = mock(StorageBackend.class);+ DynamoDbService svc = new DynamoDbService(+ new InMemoryStorage<>(), itemStore, new RegionResolver("us-east-1", "000000000000"));+ svc.createTable("Users",+ List.of(new KeySchemaElement("userId", "HASH")),+ List.of(new AttributeDefinition("userId", "S")),+ 5L, 5L, "us-east-1");+ CountDownLatch committing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ doAnswer(invocation -> {+ committing.countDown();+ release.await();+ return null;+ }).when(itemStore).put(any(), any());+ JsonNode put = mapper.readTree("""+ {"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}}}}+ """);+ JsonNode rawRequest = mapper.readTree("""+ {"ClientRequestToken":"tok-wait","TransactItems":[{"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}}}}]}+ """);++ ExecutorService executor = Executors.newFixedThreadPool(2);+ try {+ Future<DynamoDbService.TransactWriteResult> first = executor.submit(() ->+ svc.transactWriteItems(List.of(put), "us-east-1", "tok-wait", rawRequest));+ assertTrue(committing.await(5, TimeUnit.SECONDS));+ Future<DynamoDbService.TransactWriteResult> replay = executor.submit(() ->+ svc.transactWriteItems(List.of(put), "us-east-1", "tok-wait", rawRequest));+ assertThrows(TimeoutException.class, () -> replay.get(200, TimeUnit.MILLISECONDS));+ release.countDown();++ assertFalse(first.get(5, TimeUnit.SECONDS).replayed());+ DynamoDbService.TransactWriteResult replayed = replay.get(5, TimeUnit.SECONDS);+ assertTrue(replayed.replayed());+ assertEquals(2.0, replayed.capacity().get("Users").table());+ } finally {+ release.countDown();+ executor.shutdownNow();+ }+ }++ @Test+ void transactWriteItemsCancelledCallDoesNotKeepItsToken() throws Exception {+ createUsersTable("us-east-1");+ JsonNode put = mapper.readTree("""+ {"Put":{"TableName":"Users","Item":{"userId":{"S":"u1"}},"ConditionExpression":"attribute_exists(userId)"}}+ """);+ ObjectNode rawRequest = mapper.createObjectNode();+ rawRequest.putArray("TransactItems").add(put);+ rawRequest.put("ClientRequestToken", "tok-cancel");++ assertThrows(TransactionCanceledException.class, () ->+ service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest));+ assertThrows(TransactionCanceledException.class, () ->+ service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest));++ service.putItem("Users", item("userId", "u1"), "us-east-1");+ assertFalse(service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest).replayed());+ assertTrue(service.transactWriteItems(List.of(put), "us-east-1", "tok-cancel", rawRequest).replayed());+ }+@Testvoid batchWriteItem_flushesOncePerTable_notPerItem() {@SuppressWarnings("unchecked")