Light, fluffy, and always free - The AWS Local Emulator alternative
feat(dynamodb): export the changes of a period as an incremental export (#6480)
TATSUNO “Taz” Yasuhiro3dc5d356d021
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.javaindex 5545cff16c..41222924fd 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java@@ -207,7 +207,7 @@ class DynamoDbExportTest {.exportType(ExportType.INCREMENTAL_EXPORT).build())).isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class)- .hasMessageContaining("not supported");+ .hasMessageContaining("Incremental Export Specification expected when export type is set as Incremental Export");}@Test@@ -282,6 +282,24 @@ class DynamoDbExportTest {.anySatisfy(line -> assertThat(line).contains("pk:\"user-3\"", "sk:\"order-003\"", "total:15d1"));}+ @Test+ @Order(11)+ void exportTableToPointInTime_incrementalUnder15Minutes_throwsInvalidExportTimeException() {+ Instant created = ddb.describeTable(r -> r.tableName(TABLE_NAME)).table().creationDateTime();++ assertThatThrownBy(() -> ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .exportType(ExportType.INCREMENTAL_EXPORT)+ .incrementalExportSpecification(IncrementalExportSpecification.builder()+ .exportFromTime(created.truncatedTo(ChronoUnit.SECONDS))+ .exportViewType(ExportViewType.NEW_IMAGE)+ .build())+ .build()))+ .isInstanceOf(InvalidExportTimeException.class)+ .hasMessageContaining("Difference between export period from time and export period to time is less than 15 minutes");+ }+/** The manifests and the data of an export that wrote one data file. */private record ExportFiles(JsonNode manifestSummary, JsonNode dataFileEntry, String data) {}diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 0f27b2dd16..c020aa6228 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -39,7 +39,7 @@| `DescribeKinesisStreamingDestination` | List Kinesis streaming destinations || `EnableKinesisStreamingDestination` | Enable Kinesis streaming for a table || `DisableKinesisStreamingDestination` | Disable Kinesis streaming for a table |-| `ExportTableToPointInTime` | Export table data to S3 as gzip NDJSON |+| `ExportTableToPointInTime` | Export table data, or the changes of a period, to S3 || `CreateGlobalTable` | Make an existing table a global table (2017.11.29) || `DescribeGlobalTable` | Read a global table's replication group || `UpdateGlobalTable` | Add or remove replica regions |@@ -444,6 +444,19 @@ The export writes `_started`, `manifest-summary.json`, `manifest-files.json` andThe export holds the items as they were at `ExportTime`, or at the request time when `ExportTime` is not set. As on AWS, `ExportTime` must not be before `EarliestRestorableDateTime` or the table creation, and must not be in the future. A time after `LatestRestorableDateTime` is accepted.+An incremental export writes the items that changed in a period:++```bash+aws dynamodb export-table-to-point-in-time \+ --table-arn arn:aws:dynamodb:us-east-1:000000000000:table/Users \+ --s3-bucket my-exports \+ --export-type INCREMENTAL_EXPORT \+ --incremental-export-specification ExportFromTime=2026-01-01T00:00:00Z,ExportToTime=2026-01-01T01:00:00Z,ExportViewType=NEW_AND_OLD_IMAGES \+ --endpoint-url $AWS_ENDPOINT_URL+```++The period starts at `ExportFromTime` and ends just before `ExportToTime`, or at the request time when `ExportToTime` is not set. Each line is a record with `Metadata.WriteTimestampMicros`, `Keys`, and the `OldImage` and `NewImage` of the item. `ExportViewType` `NEW_IMAGE` leaves out `OldImage`. An item that ends the period as it started it, such as one inserted and then deleted, gets no record. As on AWS, the images list attribute names and set members in sorted order, and the data files go to the shared `<prefix>/AWSDynamoDB/data/` folder. As on AWS, the period must be at least 15 minutes and at most 24 hours, must not start before the table creation or `EarliestRestorableDateTime`, and must not end in the future. Floci keeps the time of a write in milliseconds, so `WriteTimestampMicros` always ends in `000`.+## Import from S3Create a new table and load it from newline-delimited DynamoDB JSON objects in S3. Each line is `{"Item": {...}}`, the format an export writes:diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.javanew file mode 100644index 0000000000..f355629216--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java@@ -0,0 +1,98 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;++import java.math.BigDecimal;+import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Base64;+import java.util.Comparator;+import java.util.List;+import java.util.Map;+import java.util.Objects;+import java.util.TreeMap;++/** The records of an incremental export. AWS sorts the attribute names and set members in them. */+final class DynamoDbExportRecords {++ private static final JsonNodeFactory NODES = JsonNodeFactory.instance;+ /** AWS sorts by code point, which String.compareTo does not do above U+FFFF. */+ private static final Comparator<String> CODE_POINT_ORDER =+ (a, b) -> Arrays.compare(a.codePoints().toArray(), b.codePoints().toArray());++ private DynamoDbExportRecords() {}++ /** A record for each item whose value at the end differs from its value at the start. */+ static List<JsonNode> records(List<ItemChange> changes, TableDefinition table, boolean newImageOnly) {+ List<String> keyNames = new ArrayList<>();+ keyNames.add(table.getPartitionKeyName());+ if (table.getSortKeyName() != null) {+ keyNames.add(table.getSortKeyName());+ }+ List<JsonNode> records = new ArrayList<>();+ for (ItemChange change : changes) {+ ObjectNode oldImage = change.oldItem() != null ? sortedAttributes(change.oldItem()) : null;+ ObjectNode newImage = change.newItem() != null ? sortedAttributes(change.newItem()) : null;+ if (Objects.equals(oldImage, newImage)) {+ continue;+ }+ JsonNode item = newImage != null ? newImage : oldImage;+ ObjectNode record = NODES.objectNode();+ record.putObject("Metadata").putObject("WriteTimestampMicros")+ .put("N", String.valueOf(ChronoUnit.MICROS.between(Instant.EPOCH, change.lastWrite())));+ ObjectNode keys = NODES.objectNode();+ for (String keyName : keyNames) {+ keys.set(keyName, item.get(keyName));+ }+ record.set("Keys", sortedAttributes(keys));+ if (oldImage != null && !newImageOnly) {+ record.set("OldImage", oldImage);+ }+ if (newImage != null) {+ record.set("NewImage", newImage);+ }+ records.add(record);+ }+ return records;+ }++ private static ObjectNode sortedAttributes(JsonNode attributes) {+ Map<String, JsonNode> byName = new TreeMap<>(CODE_POINT_ORDER);+ attributes.properties().forEach(attribute -> byName.put(attribute.getKey(), attribute.getValue()));+ ObjectNode sorted = NODES.objectNode();+ byName.forEach((name, value) -> sorted.set(name, sortedValue(value)));+ return sorted;+ }++ private static JsonNode sortedValue(JsonNode value) {+ Map.Entry<String, JsonNode> typed = value.properties().iterator().next();+ JsonNode content = typed.getValue();+ return switch (typed.getKey()) {+ case "M" -> NODES.objectNode().set("M", sortedAttributes(content));+ case "L" -> {+ ArrayNode members = NODES.arrayNode();+ content.forEach(member -> members.add(sortedValue(member)));+ yield NODES.objectNode().set("L", members);+ }+ case "SS" -> sortedSet("SS", content, Comparator.comparing(JsonNode::asText, CODE_POINT_ORDER));+ case "NS" -> sortedSet("NS", content, Comparator.comparing(member -> new BigDecimal(member.asText())));+ case "BS" -> sortedSet("BS", content, (a, b) -> Arrays.compareUnsigned(+ Base64.getDecoder().decode(a.asText()), Base64.getDecoder().decode(b.asText())));+ default -> value;+ };+ }++ private static JsonNode sortedSet(String type, JsonNode members, Comparator<JsonNode> order) {+ List<JsonNode> sorted = new ArrayList<>();+ members.forEach(sorted::add);+ sorted.sort(order);+ return NODES.objectNode().set(type, NODES.arrayNode().addAll(sorted));+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.javaindex 05161b50a1..71bada6129 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode;import java.math.BigDecimal;import java.util.HexFormat;+import java.util.List;import java.util.Map;import java.util.Set;import java.util.function.BiConsumer;@@ -19,6 +20,7 @@ final class DynamoDbIonText {/** Ion reads these as symbol ids, so they are quoted like the keywords. */private static final Pattern SYMBOL_ID = Pattern.compile("\\$[0-9]+");private static final Set<String> KEYWORDS = Set.of("null", "true", "false", "nan");+ private static final List<String> RECORD_PARTS = List.of("Keys", "Metadata", "OldImage", "NewImage");private DynamoDbIonText() {}@@ -29,6 +31,20 @@ final class DynamoDbIonText {return out.append('}').toString();}+ /** One line of an incremental export, without its newline. AWS puts Keys before Metadata here. */+ static String recordLine(JsonNode record) {+ StringBuilder out = new StringBuilder("$ion_1_0 {Record:{");+ String separator = "";+ for (String part : RECORD_PARTS) {+ if (record.has(part)) {+ out.append(separator).append(part).append(':');+ separator = ",";+ appendStruct(out, record.get(part));+ }+ }+ return out.append("}}").toString();+ }+private static void appendStruct(StringBuilder out, JsonNode attributes) {out.append('{');String separator = "";diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.javaindex 24059fb2ba..adcd128114 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java@@ -14,10 +14,13 @@ import java.util.ArrayList;import java.util.Comparator;import java.util.HashMap;import java.util.HashSet;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;+import java.util.function.BiFunction;+import java.util.function.Supplier;/*** The change history behind point-in-time recovery. A change keeps what it replaced, so a table at@@ -28,6 +31,9 @@ final class DynamoDbPointInTimeRecovery {record RestoreWindow(Instant earliest, Instant latest) {}+ /** {@code oldItem} or {@code newItem} is null when the item did not exist then. */+ record ItemChange(JsonNode oldItem, JsonNode newItem, Instant lastWrite) {}+/** The changes of one table, oldest first. */private static final class Changes {private final List<PointInTimeChange> list = new ArrayList<>();@@ -139,6 +145,61 @@ final class DynamoDbPointInTimeRecovery {return new ArrayList<>(items.values());}+ /**+ * Each item written from {@code from} until before {@code to}, in the order of its first write+ * there, with its values before and after those writes. {@code underItemLock} runs a read under+ * the lock that a writer of the item holds from recording its change until the item is written.+ */+ List<ItemChange> changesBetween(String accountId, String storageKey, Map<String, JsonNode> liveItems,+ Instant from, Instant to,+ BiFunction<String, Supplier<JsonNode>, JsonNode> underItemLock) {+ Changes changes = changesByTable.get(key(accountId, storageKey));+ if (changes == null) {+ return List.of();+ }+ Map<String, JsonNode> before = new LinkedHashMap<>();+ Map<String, Instant> lastWrites = new HashMap<>();+ Map<String, JsonNode> after = new HashMap<>();+ long seen;+ synchronized (changes) {+ seen = changes.nextSequence;+ for (PointInTimeChange change : changes.list) {+ String itemKey = change.getItemKey();+ if (change.getTime().isBefore(from)) {+ continue;+ }+ if (change.getTime().isBefore(to)) {+ if (!before.containsKey(itemKey)) {+ before.put(itemKey, change.getOldItem());+ }+ lastWrites.put(itemKey, change.getTime());+ } else if (before.containsKey(itemKey) && !after.containsKey(itemKey)) {+ after.put(itemKey, change.getOldItem());+ }+ }+ }+ List<ItemChange> result = new ArrayList<>();+ before.forEach((itemKey, oldItem) -> {+ JsonNode newItem = after.containsKey(itemKey) ? after.get(itemKey)+ : underItemLock.apply(itemKey, () -> valueBefore(changes, itemKey, seen, liveItems));+ result.add(new ItemChange(oldItem, newItem, lastWrites.get(itemKey)));+ });+ return result;+ }++ /** The value that the first change of the item from sequence {@code since} replaced, or the live item. */+ private static JsonNode valueBefore(Changes changes, String itemKey, long since, Map<String, JsonNode> liveItems) {+ synchronized (changes) {+ JsonNode value = liveItems.get(itemKey);+ for (int i = changes.list.size() - 1; 0 <= i && since <= changes.list.get(i).getSequence(); i--) {+ if (changes.list.get(i).getItemKey().equals(itemKey)) {+ value = changes.list.get(i).getOldItem();+ }+ }+ return value;+ }+ }+private void put(String accountId, String storageKey, PointInTimeChange change) {String key = storageKey + "#" + change.getSequence();if (store instanceof AccountAwareStorageBackend<PointInTimeChange> aware) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 5a8b6b1255..8fcbcbba7e 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1,9 +1,11 @@package io.github.hectorvent.floci.services.dynamodb;import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.json.JsonWriteFeature;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.ObjectWriter;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.JsonNodeFactory;import com.fasterxml.jackson.databind.node.ObjectNode;@@ -29,6 +31,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.ExportSummary;import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.ImportSummary;import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription;+import io.github.hectorvent.floci.services.dynamodb.model.IncrementalExportSpecification;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.PointInTimeChange;@@ -4921,24 +4924,43 @@ public class DynamoDbService {// --- Export Operations ---private static final List<String> EXPORT_FORMATS = List.of("ION", "DYNAMODB_JSON");+ private static final List<String> EXPORT_TYPES = List.of("FULL_EXPORT", "INCREMENTAL_EXPORT");+ private static final List<String> EXPORT_VIEW_TYPES = List.of("NEW_AND_OLD_IMAGES", "NEW_IMAGE");+ private static final List<String> S3_SSE_ALGORITHMS = List.of("AES256", "KMS");+ private static final Duration MIN_INCREMENTAL_EXPORT_PERIOD = Duration.ofMinutes(15);+ private static final Duration MAX_INCREMENTAL_EXPORT_PERIOD = Duration.ofHours(24);++ /** The IncrementalExportSpecification of a request. A member it leaves out is null. */+ public record IncrementalExportRequest(Instant from, Instant to, String viewType) {}/** As on AWS, an ExportTime after LatestRestorableDateTime is accepted up to now. */- public ExportDescription exportTable(Map<String, Object> request, Instant exportTime, String region) {+ public ExportDescription exportTable(Map<String, Object> request, Instant exportTime,+ IncrementalExportRequest incremental, String region) {String tableArn = (String) request.get("TableArn");String s3Bucket = (String) request.get("S3Bucket");String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;String exportFormat = request.get("ExportFormat") != null ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";- String exportType = request.containsKey("ExportType") ? (String) request.get("ExportType") : "FULL_EXPORT";+ String exportType = (String) request.get("ExportType");String clientToken = request.containsKey("ClientToken") ? (String) request.get("ClientToken") : null;String s3SseAlgorithm = request.containsKey("S3SseAlgorithm") ? (String) request.get("S3SseAlgorithm") : null;String s3BucketOwner = request.containsKey("S3BucketOwner") ? (String) request.get("S3BucketOwner") : null;List<String> violations = new ArrayList<>();+ if (s3SseAlgorithm != null && !S3_SSE_ALGORITHMS.contains(s3SseAlgorithm)) {+ violations.add(SmithyValidationErrors.notInEnum(s3SseAlgorithm, "s3SseAlgorithm", S3_SSE_ALGORITHMS));+ }String prefixLengthViolation = s3Prefix == null ? null: SmithyValidationErrors.lengthOutOfRange("s3Prefix", s3Prefix, 0, 1024);if (prefixLengthViolation != null) {violations.add(prefixLengthViolation);}+ if (exportType != null && !EXPORT_TYPES.contains(exportType)) {+ violations.add(SmithyValidationErrors.notInEnum(exportType, "exportType", EXPORT_TYPES));+ }+ if (incremental != null && incremental.viewType() != null && !EXPORT_VIEW_TYPES.contains(incremental.viewType())) {+ violations.add(SmithyValidationErrors.notInEnum(incremental.viewType(),+ "incrementalExportSpecification.exportViewType", EXPORT_VIEW_TYPES));+ }if (!EXPORT_FORMATS.contains(exportFormat)) {violations.add(SmithyValidationErrors.notInEnum(exportFormat, "exportFormat", EXPORT_FORMATS));}@@ -4951,10 +4973,6 @@ public class DynamoDbService {throw new AwsException("ValidationException", "Invalid Request: "+ SmithyValidationErrors.message(List.of(prefixLengthViolation)), 400);}- if ("INCREMENTAL_EXPORT".equals(exportType)) {- throw new AwsException("ValidationException",- "ExportType INCREMENTAL_EXPORT is not supported", 400);- }DynamoDbTableNames.ResolvedTableRef ref = DynamoDbTableNames.resolveWithRegion(tableArn, region);String tableName = ref.name();@@ -4962,24 +4980,49 @@ public class DynamoDbService {String storageKey = regionKey(tableRegion, tableName);TableDefinition table = requireBackupSource(storageKey, tableArn);+ boolean incrementalExport = "INCREMENTAL_EXPORT".equals(exportType);+ if (incrementalExport && incremental == null) {+ throw invalidExportRequest("Incremental Export Specification expected when export type is set as Incremental Export");+ }+ if (!incrementalExport && incremental != null) {+ throw invalidExportRequest(exportType == null+ ? "Export Type expected to be Incremental Export when incremental export specification is provided"+ : "When Export Type is Full Export , Incremental Export Specification should not be provided");+ }+ if (incrementalExport && incremental.from() == null) {+ throw invalidExportRequest("ExportFromTime must be provided in IncrementalExportSpecification");+ }if (s3Prefix != null && 948 < s3Prefix.getBytes(StandardCharsets.UTF_8).length) {throw new AwsException("ValidationException", "Invalid Request: S3 Prefix is too long", 400);}- Instant startTime = clock.instant().truncatedTo(ChronoUnit.MILLIS);+ Instant now = clock.instant();// AWS cannot enable point-in-time recovery until seconds after the creation, so its// EarliestRestorableDateTime is never in the creation second.- if (exportTime != null && (exportTime.isBefore(table.getCreationDateTime().truncatedTo(ChronoUnit.SECONDS))- || startTime.isBefore(exportTime))) {- throw new AwsException("InvalidExportTimeException", "Export Time is invalid", 400);+ Instant creationSecond = table.getCreationDateTime().truncatedTo(ChronoUnit.SECONDS);+ Instant startTime;+ Instant exportFromTime = null;+ Instant exportToTime = null;+ if (incrementalExport) {+ startTime = now.truncatedTo(ChronoUnit.SECONDS);+ exportFromTime = incremental.from();+ exportToTime = incremental.to() != null ? incremental.to() : startTime;+ checkIncrementalExportPeriod(exportFromTime, exportToTime, creationSecond, now);+ } else {+ startTime = now.truncatedTo(ChronoUnit.MILLIS);+ if (exportTime != null && (exportTime.isBefore(creationSecond) || startTime.isBefore(exportTime))) {+ throw new AwsException("InvalidExportTimeException", "Export Time is invalid", 400);+ }}if (!table.isPointInTimeRecoveryEnabled()) {throw new AwsException("PointInTimeRecoveryUnavailableException","Point in time recovery is not enabled for table '" + table.getTableName() + "'", 400);}- if (exportTime != null- && exportTime.isBefore(pointInTimeRecovery.window(withPointInTimeStart(storageKey, table)).earliest())) {- throw new AwsException("InvalidExportTimeException",- "Point in Time Recovery is not available for specified export time", 400);+ Instant readFrom = incrementalExport ? exportFromTime : exportTime;+ if (readFrom != null+ && readFrom.isBefore(pointInTimeRecovery.window(withPointInTimeStart(storageKey, table)).earliest())) {+ throw new AwsException("InvalidExportTimeException", incrementalExport+ ? "Export period from time cannot be earlier than the Point in Time Recovery window"+ : "Point in Time Recovery is not available for specified export time", 400);}Instant time = exportTime != null ? exportTime : startTime;@@ -4994,8 +5037,13 @@ public class DynamoDbService {desc.setS3Bucket(s3Bucket);desc.setS3Prefix(s3Prefix);desc.setExportFormat(exportFormat);- desc.setExportType((String) request.get("ExportType"));- desc.setExportTime(epochSeconds(time));+ desc.setExportType(exportType);+ if (incrementalExport) {+ desc.setIncrementalExportSpecification(new IncrementalExportSpecification(epochSeconds(exportFromTime),+ epochSeconds(exportToTime), incremental.viewType() != null ? incremental.viewType() : "NEW_AND_OLD_IMAGES"));+ } else {+ desc.setExportTime(epochSeconds(time));+ }desc.setStartTime(epochSeconds(startTime));desc.setClientToken(clientToken);desc.setS3SseAlgorithm(s3SseAlgorithm != null ? s3SseAlgorithm : "AES256");@@ -5006,8 +5054,13 @@ public class DynamoDbService {}ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(scopedItemsKey(storageKey));- List<JsonNode> snapshot = pointInTimeRecovery.itemsAt(regionResolver.getAccountId(), storageKey,- tableItems != null ? tableItems : Map.of(), time);+ Map<String, JsonNode> liveItems = tableItems != null ? tableItems : Map.of();+ List<JsonNode> snapshot = incrementalExport+ ? DynamoDbExportRecords.records(pointInTimeRecovery.changesBetween(regionResolver.getAccountId(),+ storageKey, liveItems, exportFromTime, exportToTime,+ (itemKey, read) -> withItemLock(storageKey, itemKey, read)), table,+ "NEW_IMAGE".equals(incremental.viewType()))+ : pointInTimeRecovery.itemsAt(regionResolver.getAccountId(), storageKey, liveItems, time);// The worker gets its own copy so the object serialized into this response never// changes underneath the handler, and the request account so its writes land there.@@ -5018,14 +5071,16 @@ public class DynamoDbService {return desc;}+ /** AWS puts an incremental export's data files in a data folder that all its exports share. */private void runExport(ExportDescription desc, List<JsonNode> snapshot, String exportArn) {try {String s3Bucket = desc.getS3Bucket();String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);String prefix = desc.getS3Prefix();// AWS adds no slash after a prefix that ends with one.- String exportDir = (prefix == null ? "" : prefix.endsWith("/") ? prefix : prefix + "/")- + "AWSDynamoDB/" + exportId + "/";+ String exportRoot = (prefix == null ? "" : prefix.endsWith("/") ? prefix : prefix + "/") + "AWSDynamoDB/";+ String exportDir = exportRoot + exportId + "/";+ boolean incremental = desc.getIncrementalExportSpecification() != null;try {writeExportFile(s3Bucket, exportDir + "_started", new byte[0]);@@ -5044,8 +5099,9 @@ public class DynamoDbService {}boolean ion = "ION".equals(desc.getExportFormat());- byte[] gzipData = buildGzipExportData(snapshot, ion);- String dataKey = exportDir + "data/" + exportFileName() + (ion ? ".ion.gz" : ".json.gz");+ byte[] gzipData = buildGzipExportData(snapshot, ion, incremental);+ String dataKey = (incremental ? exportRoot : exportDir) + "data/" + exportFileName()+ + (ion ? ".ion.gz" : ".json.gz");ObjectNode dataFile = objectMapper.createObjectNode();dataFile.put("itemCount", snapshot.size());dataFile.put("md5Checksum", Base64.getEncoder().encodeToString(Digests.md5(gzipData)));@@ -5056,7 +5112,8 @@ public class DynamoDbService {desc.setEndTime(epochSeconds(clock.instant()));desc.setItemCount((long) snapshot.size());- desc.setBilledSizeBytes((long) gzipData.length);+ // AWS bills an incremental export for at least 10 MB.+ desc.setBilledSizeBytes(incremental ? Math.max(10_000_000L, gzipData.length) : gzipData.length);String manifestSummaryKey = exportDir + "manifest-summary.json";writeExportManifest(s3Bucket, manifestSummaryKey, manifestSummary(desc, manifestFilesKey));@@ -5079,16 +5136,21 @@ public class DynamoDbService {}}- private byte[] buildGzipExportData(List<JsonNode> items, boolean ion) throws IOException {+ private byte[] buildGzipExportData(List<JsonNode> items, boolean ion, boolean incremental) throws IOException {+ // AWS writes a character above U+FFFF as UTF-8, where Jackson writes two escapes by default.+ ObjectWriter lineWriter = objectMapper.writer().with(JsonWriteFeature.COMBINE_UNICODE_SURROGATES_IN_UTF8);ByteArrayOutputStream baos = new ByteArrayOutputStream();try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {for (JsonNode item : items) {if (ion) {- gzip.write(DynamoDbIonText.exportLine(item).getBytes(StandardCharsets.UTF_8));+ String line = incremental ? DynamoDbIonText.recordLine(item) : DynamoDbIonText.exportLine(item);+ gzip.write(line.getBytes(StandardCharsets.UTF_8));+ } else if (incremental) {+ gzip.write(lineWriter.writeValueAsBytes(item));} else {ObjectNode line = objectMapper.createObjectNode();line.set("Item", item);- byte[] lineBytes = objectMapper.writeValueAsBytes(line);+ byte[] lineBytes = lineWriter.writeValueAsBytes(line);gzip.write(lineBytes);}gzip.write('\n');@@ -5134,14 +5196,20 @@ public class DynamoDbService {}private String manifestSummary(ExportDescription desc, String manifestFilesKey) throws JsonProcessingException {+ IncrementalExportSpecification incremental = desc.getIncrementalExportSpecification();ObjectNode root = objectMapper.createObjectNode();- root.put("version", "2020-06-30");+ root.put("version", incremental != null ? "2023-08-01" : "2020-06-30");root.put("exportArn", desc.getExportArn());root.put("startTime", MANIFEST_TIME.format(instantOf(desc.getStartTime())));root.put("endTime", MANIFEST_TIME.format(instantOf(desc.getEndTime())));root.put("tableArn", desc.getTableArn());root.put("tableId", desc.getTableId());- root.put("exportTime", MANIFEST_TIME.format(instantOf(desc.getExportTime())));+ if (incremental != null) {+ root.put("exportFromTime", MANIFEST_TIME.format(instantOf(incremental.getExportFromTime())));+ root.put("exportToTime", MANIFEST_TIME.format(instantOf(incremental.getExportToTime())));+ } else {+ root.put("exportTime", MANIFEST_TIME.format(instantOf(desc.getExportTime())));+ }root.put("s3Bucket", desc.getS3Bucket());root.put("s3Prefix", desc.getS3Prefix());root.put("s3SseAlgorithm", desc.getS3SseAlgorithm());@@ -5150,9 +5218,42 @@ public class DynamoDbService {root.put("billedSizeBytes", desc.getBilledSizeBytes());root.put("itemCount", desc.getItemCount());root.put("outputFormat", desc.getExportFormat());+ if (incremental != null) {+ root.put("outputView", incremental.getExportViewType());+ root.put("exportType", desc.getExportType());+ }return objectMapper.writeValueAsString(root);}+ private static void checkIncrementalExportPeriod(Instant from, Instant to, Instant creationSecond, Instant now) {+ if (from.isBefore(creationSecond)) {+ throw new AwsException("InvalidExportTimeException",+ "Incremental export period from time cannot be less than the table creation time : " + from, 400);+ }+ if (to.isAfter(now)) {+ throw new AwsException("InvalidExportTimeException",+ "Incremental export period to time cannot be greater than the current time : " + to, 400);+ }+ if (!from.isBefore(to)) {+ throw new AwsException("InvalidExportTimeException",+ "Incremental export period from time should not be greater or equal to incremental export period to time",+ 400);+ }+ if (Duration.between(from, to).compareTo(MIN_INCREMENTAL_EXPORT_PERIOD) < 0) {+ throw new AwsException("InvalidExportTimeException",+ "Difference between export period from time and export period to time is less than 15 minutes", 400);+ }+ // AWS takes 24 h + 1 ms but not 24 h + 1 s.+ if (Duration.between(from, to).toSeconds() > MAX_INCREMENTAL_EXPORT_PERIOD.toSeconds()) {+ throw new AwsException("InvalidExportTimeException",+ "Difference between export period from time and export period to time is greater than 24 hours", 400);+ }+ }++ private static AwsException invalidExportRequest(String message) {+ return new AwsException("ValidationException", "Invalid Request: " + message, 400);+ }+public ExportDescription describeExport(String exportArn) {if (exportStore == null) {throw new AwsException("ExportNotFoundException",diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex adb248068b..5828fa80c8 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -3405,8 +3405,13 @@ public class NativeDynamoDbJsonHandler {request.fields().forEachRemaining(e -> params.put(e.getKey(), e.getValue().isTextual()? e.getValue().asText() : e.getValue()));+ JsonNode spec = request.path("IncrementalExportSpecification");+ DynamoDbService.IncrementalExportRequest incremental = spec.isObject()+ ? new DynamoDbService.IncrementalExportRequest(timestampOrNull(spec, "ExportFromTime"),+ timestampOrNull(spec, "ExportToTime"), spec.path("ExportViewType").textValue())+ : null;io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =- dynamoDbService.exportTable(params, timestampOrNull(request, "ExportTime"), region);+ dynamoDbService.exportTable(params, timestampOrNull(request, "ExportTime"), incremental, region);ObjectNode response = objectMapper.createObjectNode();response.set("ExportDescription", objectMapper.valueToTree(desc));diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.javaindex 682d46c8a3..e62de24fa7 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java@@ -39,6 +39,9 @@ public class ExportDescription {@JsonProperty("ExportTime")private BigDecimal exportTime;+ @JsonProperty("IncrementalExportSpecification")+ private IncrementalExportSpecification incrementalExportSpecification;+@JsonProperty("StartTime")private BigDecimal startTime;@@ -99,6 +102,11 @@ public class ExportDescription {public BigDecimal getExportTime() { return exportTime; }public void setExportTime(BigDecimal exportTime) { this.exportTime = exportTime; }+ public IncrementalExportSpecification getIncrementalExportSpecification() { return incrementalExportSpecification; }+ public void setIncrementalExportSpecification(IncrementalExportSpecification incrementalExportSpecification) {+ this.incrementalExportSpecification = incrementalExportSpecification;+ }+public BigDecimal getStartTime() { return startTime; }public void setStartTime(BigDecimal startTime) { this.startTime = startTime; }diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/IncrementalExportSpecification.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/IncrementalExportSpecification.javanew file mode 100644index 0000000000..e5d4052172--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/IncrementalExportSpecification.java@@ -0,0 +1,40 @@+package io.github.hectorvent.floci.services.dynamodb.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.math.BigDecimal;++@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+@JsonInclude(JsonInclude.Include.NON_NULL)+public class IncrementalExportSpecification {++ @JsonProperty("ExportFromTime")+ private BigDecimal exportFromTime;++ @JsonProperty("ExportToTime")+ private BigDecimal exportToTime;++ @JsonProperty("ExportViewType")+ private String exportViewType;++ public IncrementalExportSpecification() {}++ public IncrementalExportSpecification(BigDecimal exportFromTime, BigDecimal exportToTime, String exportViewType) {+ this.exportFromTime = exportFromTime;+ this.exportToTime = exportToTime;+ this.exportViewType = exportViewType;+ }++ public BigDecimal getExportFromTime() { return exportFromTime; }+ public void setExportFromTime(BigDecimal exportFromTime) { this.exportFromTime = exportFromTime; }++ public BigDecimal getExportToTime() { return exportToTime; }+ public void setExportToTime(BigDecimal exportToTime) { this.exportToTime = exportToTime; }++ public String getExportViewType() { return exportViewType; }+ public void setExportViewType(String exportViewType) { this.exportViewType = exportViewType; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.javaindex cf7eef7b51..6c0bfadd4a 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java@@ -158,21 +158,48 @@ class DynamoDbExportIntegrationTest {}@Test- void exportTableToPointInTime_unsupportedExportType_returnsValidationException() {- given()- .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")- .contentType(DYNAMODB_CONTENT_TYPE)- .body("""- {- "TableArn": "%s",- "S3Bucket": "%s",- "ExportType": "INCREMENTAL_EXPORT"- }- """.formatted(TABLE_ARN, BUCKET_NAME))- .when().post("/")- .then()- .statusCode(400)- .body("__type", containsString("ValidationException"));+ void incrementalExportRequestIsCheckedAsAwsChecksIt() {+ String over1024 = "a".repeat(1025);+ String enumSet = "' failed to satisfy constraint: Member must satisfy enum value set: ";+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3SseAlgorithm": "BAD", "S3Prefix": "%s", "ExportType": "BAD",+ "IncrementalExportSpecification": {"ExportViewType": "BAD"}, "ExportFormat": "BAD"}+ """.formatted(TABLE_ARN_PREFIX + "IncrementalMissing", BUCKET_NAME, over1024)), "ValidationException",+ "5 validation errors detected: Value 'BAD' at 's3SseAlgorithm" + enumSet + "[AES256, KMS]; Value '"+ + over1024 + "' at 's3Prefix' failed to satisfy constraint: Member must have length less than or equal to 1024; "+ + "Value 'BAD' at 'exportType" + enumSet + "[FULL_EXPORT, INCREMENTAL_EXPORT]; "+ + "Value 'BAD' at 'incrementalExportSpecification.exportViewType" + enumSet + "[NEW_AND_OLD_IMAGES, NEW_IMAGE]; "+ + "Value 'BAD' at 'exportFormat" + enumSet + "[ION, DYNAMODB_JSON]");+ expectError(incrementalExport("IncrementalMissing", "inc", null), "TableNotFoundException",+ "Table not found: " + TABLE_ARN_PREFIX + "IncrementalMissing");++ String tooLong = "a".repeat(949);+ expectError(incrementalExport(TABLE_NAME, tooLong, null), "ValidationException",+ "Invalid Request: Incremental Export Specification expected when export type is set as Incremental Export");+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "%s", "ExportType": "FULL_EXPORT",+ "IncrementalExportSpecification": {"ExportFromTime": 1}}+ """.formatted(TABLE_ARN, BUCKET_NAME, tooLong)), "ValidationException",+ "Invalid Request: When Export Type is Full Export , Incremental Export Specification should not be provided");+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "%s", "IncrementalExportSpecification": {"ExportFromTime": 1}}+ """.formatted(TABLE_ARN, BUCKET_NAME, tooLong)), "ValidationException",+ "Invalid Request: Export Type expected to be Incremental Export when incremental export specification is provided");+ expectError(incrementalExport(TABLE_NAME, tooLong, "{\"ExportToTime\": 1}"), "ValidationException",+ "Invalid Request: ExportFromTime must be provided in IncrementalExportSpecification");+ expectError(incrementalExport(TABLE_NAME, tooLong, "{\"ExportFromTime\": 1}"), "ValidationException",+ "Invalid Request: S3 Prefix is too long");+ expectError(incrementalExport(TABLE_NAME, "inc", "{\"ExportFromTime\": 1000000000.5, \"ExportToTime\": 1}"),+ "InvalidExportTimeException",+ "Incremental export period from time cannot be less than the table creation time : 2001-09-09T01:46:40.500Z");+ long now = System.currentTimeMillis() / 1000;+ expectError(incrementalExport(TABLE_NAME, "inc", "{\"ExportFromTime\": %d, \"ExportToTime\": %d}"+ .formatted(now, now + 3600)), "InvalidExportTimeException",+ "Incremental export period to time cannot be greater than the current time : "+ + Instant.ofEpochSecond(now + 3600));+ expectError(incrementalExport(TABLE_NAME, "inc", "{\"ExportFromTime\": %d}".formatted(now + 60)),+ "InvalidExportTimeException",+ "Incremental export period from time should not be greater or equal to incremental export period to time");}@Test@@ -398,6 +425,13 @@ class DynamoDbExportIntegrationTest {// --- Helpers ---+ private static ValidatableResponse incrementalExport(String table, String prefix, String specification) {+ return ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "%s", "ExportType": "INCREMENTAL_EXPORT"%s}+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME, prefix,+ specification != null ? ", \"IncrementalExportSpecification\": " + specification : ""));+ }+private static ValidatableResponse exportWithPrefix(String table, String prefix) {return given().header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecordsTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecordsTest.javanew file mode 100644index 0000000000..17e64bc18e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecordsTest.java@@ -0,0 +1,80 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.time.Instant;+import java.time.temporal.ChronoUnit;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;++class DynamoDbExportRecordsTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();+ private static final TableDefinition TABLE = new TableDefinition("Sorted",+ List.of(new KeySchemaElement("zk", "HASH"), new KeySchemaElement("ak", "RANGE")),+ List.of(new AttributeDefinition("zk", "S"), new AttributeDefinition("ak", "N")));+ private static final String ITEM = """+ {"zk": {"S": "old"}, "ak": {"N": "1"}, "😀": {"S": "emoji name"}, "~": {"S": "fullwidth tilde name"},+ "z": {"S": "z"}, "a": {"S": "a"}, "B": {"S": "upper B"}, "é": {"S": "e acute"},+ "ss": {"SS": ["😀", "~", "b", "B", "a", "é"]}, "ns": {"NS": ["3", "-10", "100", "-2.5", "0", "0.25"]},+ "bs": {"BS": ["/w==", "AA==", "gA==", "fw==", "Pg==", "AAE="]},+ "m": {"M": {"z": {"N": "1"}, "a": {"N": "2"}, "😀": {"S": "e"}, "~": {"S": "t"},+ "nested": {"M": {"y": {"N": "1"}, "b": {"N": "2"}}}}},+ "l": {"L": [{"M": {"z": {"N": "1"}, "a": {"N": "2"}}}, {"SS": ["b", "a"]}, {"NS": ["2", "1"]}]},+ "v": {"S": "%s"}}+ """;+ private static final String SORTED = "\"B\":{\"S\":\"upper B\"},\"a\":{\"S\":\"a\"},\"ak\":{\"N\":\"1\"},"+ + "\"bs\":{\"BS\":[\"AA==\",\"AAE=\",\"Pg==\",\"fw==\",\"gA==\",\"/w==\"]},"+ + "\"l\":{\"L\":[{\"M\":{\"a\":{\"N\":\"2\"},\"z\":{\"N\":\"1\"}}},{\"SS\":[\"a\",\"b\"]},{\"NS\":[\"1\",\"2\"]}]},"+ + "\"m\":{\"M\":{\"a\":{\"N\":\"2\"},\"nested\":{\"M\":{\"b\":{\"N\":\"2\"},\"y\":{\"N\":\"1\"}}},"+ + "\"z\":{\"N\":\"1\"},\"~\":{\"S\":\"t\"},\"😀\":{\"S\":\"e\"}}},"+ + "\"ns\":{\"NS\":[\"-10\",\"-2.5\",\"0\",\"0.25\",\"3\",\"100\"]},"+ + "\"ss\":{\"SS\":[\"B\",\"a\",\"b\",\"é\",\"~\",\"😀\"]},\"v\":{\"S\":\"%s\"},\"z\":{\"S\":\"z\"},"+ + "\"zk\":{\"S\":\"old\"},\"é\":{\"S\":\"e acute\"},\"~\":{\"S\":\"fullwidth tilde name\"},"+ + "\"😀\":{\"S\":\"emoji name\"}";+ private static final String ION_SORTED = "B:\"upper B\",a:\"a\",ak:1.,"+ + "bs:$dynamodb_BS::[{{AA==}},{{AAE=}},{{Pg==}},{{fw==}},{{gA==}},{{/w==}}],"+ + "l:[{a:2.,z:1.},$dynamodb_SS::[\"a\",\"b\"],$dynamodb_NS::[1.,2.]],"+ + "m:{a:2.,nested:{b:2.,y:1.},z:1.,'~':\"t\",'😀':\"e\"},ns:$dynamodb_NS::[-1d1,-2.5,0.,0.25,3.,1d2],"+ + "ss:$dynamodb_SS::[\"B\",\"a\",\"b\",\"é\",\"~\",\"😀\"],v:\"%s\",z:\"z\",zk:\"old\",'é':\"e acute\","+ + "'~':\"fullwidth tilde name\",'😀':\"emoji name\"";++ @Test+ void recordIsTheRecordAwsWrites() throws IOException {+ ItemChange update = new ItemChange(MAPPER.readTree(ITEM.formatted("before")), MAPPER.readTree(ITEM.formatted("after")),+ Instant.EPOCH.plus(1792884188164874L, ChronoUnit.MICROS));++ JsonNode record = DynamoDbExportRecords.records(List.of(update), TABLE, false).getFirst();++ assertEquals("{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"1792884188164874\"}},"+ + "\"Keys\":{\"ak\":{\"N\":\"1\"},\"zk\":{\"S\":\"old\"}},\"OldImage\":{" + SORTED.formatted("before")+ + "},\"NewImage\":{" + SORTED.formatted("after") + "}}", MAPPER.writeValueAsString(record));+ assertEquals("$ion_1_0 {Record:{Keys:{ak:1.,zk:\"old\"},Metadata:{WriteTimestampMicros:1792884188164874.},"+ + "OldImage:{" + ION_SORTED.formatted("before") + "},NewImage:{" + ION_SORTED.formatted("after") + "}}}",+ DynamoDbIonText.recordLine(record));+ }++ @Test+ void itemWhoseSetsOnlyChangeOrderGetsNoRecord() throws IOException {+ String item = """+ {"zk": {"S": "k"}, "ak": {"N": "1"}, "ss": {"SS": %s}, "ns": {"NS": %s}, "bs": {"BS": %s},+ "m": {"M": {"ss": {"SS": %s}}}, "l": {"L": [{"NS": %s}]}}+ """;+ ItemChange reordered = new ItemChange(+ MAPPER.readTree(item.formatted("[\"a\", \"b\"]", "[\"1\", \"2\"]", "[\"AQ==\", \"Ag==\"]",+ "[\"x\", \"y\"]", "[\"10\", \"20\"]")),+ MAPPER.readTree(item.formatted("[\"b\", \"a\"]", "[\"2\", \"1\"]", "[\"Ag==\", \"AQ==\"]",+ "[\"y\", \"x\"]", "[\"20\", \"10\"]")),+ Instant.EPOCH);++ assertEquals(List.of(), DynamoDbExportRecords.records(List.of(reordered), TABLE, false));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.javanew file mode 100644index 0000000000..9cacd89d1c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.java@@ -0,0 +1,78 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.time.Instant;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;++import static org.junit.jupiter.api.Assertions.assertEquals;++class DynamoDbPointInTimeRecoveryTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();+ private static final String ACCOUNT = "000000000000";+ private static final String TABLE_KEY = "us-east-1::Items";++ private final MutableClock clock = new MutableClock();+ private final DynamoDbPointInTimeRecovery recovery =+ new DynamoDbPointInTimeRecovery(new InMemoryStorage<>(), ACCOUNT, Duration.ofMinutes(5), clock);+ private final TableDefinition table = new TableDefinition("Items",+ List.of(new KeySchemaElement("pk", "HASH")), List.of(new AttributeDefinition("pk", "S")));+ private final Map<String, JsonNode> live = new ConcurrentHashMap<>();++ @Test+ void changesBetweenWaitsForAWriteThatIsRecordedButNotYetWritten() {+ table.setPointInTimeRecoveryEnabled(true);+ Instant from = clock.instant();+ live.put("k", item("A"));+ recovery.recordItem(ACCOUNT, TABLE_KEY, table, "k", item("A"));++ List<ItemChange> changes = recovery.changesBetween(ACCOUNT, TABLE_KEY, live, from, from.plusSeconds(60),+ (itemKey, read) -> {+ live.put(itemKey, item("B"));+ return read.get();+ });++ assertEquals(1, changes.size());+ assertEquals(item("A"), changes.getFirst().oldItem());+ assertEquals(item("B"), changes.getFirst().newItem());+ }++ @Test+ void changesBetweenLeavesOutAWriteAfterTheEndThatFinishesBeforeTheItemIsRead() {+ table.setPointInTimeRecoveryEnabled(true);+ Instant from = clock.instant();+ recovery.recordItem(ACCOUNT, TABLE_KEY, table, "k", item("A"));+ live.put("k", item("B"));++ List<ItemChange> changes = recovery.changesBetween(ACCOUNT, TABLE_KEY, live, from, from.plusSeconds(60),+ (itemKey, read) -> {+ clock.advance(Duration.ofMinutes(2));+ recovery.recordItem(ACCOUNT, TABLE_KEY, table, itemKey, live.get(itemKey));+ live.put(itemKey, item("C"));+ return read.get();+ });++ assertEquals(1, changes.size());+ assertEquals(item("B"), changes.getFirst().newItem());+ }++ private static JsonNode item(String value) {+ ObjectNode item = MAPPER.createObjectNode();+ item.putObject("pk").put("S", "k");+ item.putObject("v").put("S", value);+ return item;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex 261fe847e3..3f50778062 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -33,13 +33,17 @@ import org.mockito.ArgumentCaptor;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.math.BigDecimal;import java.nio.charset.StandardCharsets;import java.nio.file.Path;import java.time.Duration;import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Arrays;import java.util.HashSet;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Set;@@ -4509,7 +4513,7 @@ class DynamoDbServiceTest {Map<String, Object> request = Map.of("TableArn", tableArn, "S3Bucket", "bucket");AwsException e = assertThrows(AwsException.class,- () -> service.exportTable(request, Instant.EPOCH, "us-east-1"));+ () -> service.exportTable(request, Instant.EPOCH, null, "us-east-1"));assertEquals("TableNotFoundException", e.getErrorCode());assertEquals("Table not found: " + tableArn, e.getMessage());@@ -4524,7 +4528,7 @@ class DynamoDbServiceTest {.thenThrow(new AwsException("InternalError", "We encountered an internal error. Please try again.", 500));DynamoDbService svc = inlineExportService(s3);- String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket"), null, "us-east-1")+ String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket"), null, null, "us-east-1").getExportArn();verify(s3).abortMultipartUpload(eq("bucket"), endsWith("/_started"), eq(upload.getUploadId()));@@ -4538,8 +4542,8 @@ class DynamoDbServiceTest {String longest = "a".repeat(948);String slashed = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", "ends-with/"),- null, "us-east-1").getExportArn();- svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", longest), null, "us-east-1");+ null, null, "us-east-1").getExportArn();+ svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", longest), null, null, "us-east-1");String slashedId = slashed.substring(slashed.lastIndexOf('/') + 1);verify(s3).initiateMultipartUpload("bucket", "ends-with/AWSDynamoDB/" + slashedId + "/_started",@@ -4558,24 +4562,198 @@ class DynamoDbServiceTest {svc.putItem("Users", mapper.readTree("{\"userId\": {\"S\": \"u1\"}, \"total\": {\"N\": \"150\"}}"), "us-east-1");String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", "exports-ion",- "ExportFormat", "ION"), null, "us-east-1").getExportArn();+ "ExportFormat", "ION"), null, null, "us-east-1").getExportArn();ExportDescription described = svc.describeExport(exportArn);assertEquals("COMPLETED", described.getExportStatus());assertEquals("ION", described.getExportFormat());+ Map<String, byte[]> files = uploadedFiles(s3);+ String dir = "exports-ion/AWSDynamoDB/" + exportArn.substring(exportArn.lastIndexOf('/') + 1) + "/";+ String dataKey = files.keySet().stream().filter(key -> key.endsWith(".ion.gz")).findFirst().orElseThrow();+ assertTrue(dataKey.matches(dir + "data/[a-z2-7]{26}\\.ion\\.gz"), dataKey);+ assertEquals("$ion_1_0 {Item:{userId:\"u1\",total:15d1}}\n", gunzip(files.get(dataKey)));+ assertEquals("ION", mapper.readTree(files.get(dir + "manifest-summary.json")).get("outputFormat").asText());+ }++ @Test+ void jsonExportWritesCharactersAboveUffffAsUtf8() throws Exception {+ S3Service s3 = exportBucket();+ DynamoDbService svc = inlineExportService(s3);+ svc.putItem("Users", item("userId", "u1", "😀", "😀"), "us-east-1");++ svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket"), null, null, "us-east-1");++ byte[] dataFile = uploadedFiles(s3).entrySet().stream().filter(file -> file.getKey().endsWith(".json.gz"))+ .findFirst().orElseThrow().getValue();+ assertEquals("{\"Item\":{\"userId\":{\"S\":\"u1\"},\"😀\":{\"S\":\"😀\"}}}\n", gunzip(dataFile));+ }++ @Test+ void incrementalExportWritesARecordForEachChangedItem() throws Exception {+ S3Service s3 = exportBucket();+ DynamoDbService svc = incrementalExportService(s3);+ for (String id : List.of("updated", "deleted", "same", "twice", "deletedThenPut")) {+ svc.putItem("Users", item("userId", id, "v", "before"), "us-east-1");+ }+ clock.advance(Duration.ofSeconds(1));+ Instant from = clock.instant();+ svc.putItem("Users", item("userId", "updated", "v", "after"), "us-east-1");+ svc.deleteItem("Users", item("userId", "deleted"), "us-east-1");+ svc.putItem("Users", item("userId", "new", "v", "new 😀"), "us-east-1");+ svc.putItem("Users", item("userId", "newThenDeleted", "v", "new"), "us-east-1");+ svc.deleteItem("Users", item("userId", "newThenDeleted"), "us-east-1");+ svc.putItem("Users", item("userId", "same", "v", "before"), "us-east-1");+ svc.putItem("Users", item("userId", "twice", "v", "first"), "us-east-1");+ Instant beforeLastWrite = clock.instant();+ svc.putItem("Users", item("userId", "twice", "v", "second"), "us-east-1");+ svc.deleteItem("Users", item("userId", "deletedThenPut"), "us-east-1");+ svc.putItem("Users", item("userId", "deletedThenPut", "v", "put"), "us-east-1");+ clock.advance(Duration.ofMinutes(15));+ Instant to = clock.instant();+ svc.putItem("Users", item("userId", "updated", "v", "after the period"), "us-east-1");++ String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", "inc",+ "ExportType", "INCREMENTAL_EXPORT"), null, new DynamoDbService.IncrementalExportRequest(from, to, null),+ "us-east-1").getExportArn();++ assertTrue(exportArn.matches(".*/export/\\d{11}000-[0-9a-f]{8}"), exportArn);+ ExportDescription described = svc.describeExport(exportArn);+ assertEquals("COMPLETED", described.getExportStatus());+ assertEquals("INCREMENTAL_EXPORT", described.getExportType());+ assertNull(described.getExportTime());+ assertEquals(BigDecimal.valueOf(from.toEpochMilli(), 3),+ described.getIncrementalExportSpecification().getExportFromTime());+ assertEquals(BigDecimal.valueOf(to.toEpochMilli(), 3), described.getIncrementalExportSpecification().getExportToTime());+ assertEquals("NEW_AND_OLD_IMAGES", described.getIncrementalExportSpecification().getExportViewType());+ Map<String, byte[]> files = uploadedFiles(s3);+ String dataKey = files.keySet().stream().filter(key -> key.endsWith(".json.gz")).findFirst().orElseThrow();+ assertTrue(dataKey.matches("inc/AWSDynamoDB/data/[a-z2-7]{26}\\.json\\.gz"), dataKey);+ String[] records = gunzip(files.get(dataKey)).split("\n");+ assertEquals(List.of(+ "{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"T\"}},\"Keys\":{\"userId\":{\"S\":\"updated\"}},"+ + "\"OldImage\":{\"userId\":{\"S\":\"updated\"},\"v\":{\"S\":\"before\"}},"+ + "\"NewImage\":{\"userId\":{\"S\":\"updated\"},\"v\":{\"S\":\"after\"}}}",+ "{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"T\"}},\"Keys\":{\"userId\":{\"S\":\"deleted\"}},"+ + "\"OldImage\":{\"userId\":{\"S\":\"deleted\"},\"v\":{\"S\":\"before\"}}}",+ "{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"T\"}},\"Keys\":{\"userId\":{\"S\":\"new\"}},"+ + "\"NewImage\":{\"userId\":{\"S\":\"new\"},\"v\":{\"S\":\"new 😀\"}}}",+ "{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"T\"}},\"Keys\":{\"userId\":{\"S\":\"twice\"}},"+ + "\"OldImage\":{\"userId\":{\"S\":\"twice\"},\"v\":{\"S\":\"before\"}},"+ + "\"NewImage\":{\"userId\":{\"S\":\"twice\"},\"v\":{\"S\":\"second\"}}}",+ "{\"Metadata\":{\"WriteTimestampMicros\":{\"N\":\"T\"}},\"Keys\":{\"userId\":{\"S\":\"deletedThenPut\"}},"+ + "\"OldImage\":{\"userId\":{\"S\":\"deletedThenPut\"},\"v\":{\"S\":\"before\"}},"+ + "\"NewImage\":{\"userId\":{\"S\":\"deletedThenPut\"},\"v\":{\"S\":\"put\"}}}"),+ Arrays.stream(records).map(r -> r.replaceFirst("\"N\":\"\\d+\"", "\"N\":\"T\"")).toList());+ assertEquals(5L, described.getItemCount());+ assertEquals(10_000_000L, described.getBilledSizeBytes());+ long lastWriteMicros = mapper.readTree(records[3]).at("/Metadata/WriteTimestampMicros/N").asLong();+ assertTrue(ChronoUnit.MICROS.between(Instant.EPOCH, beforeLastWrite) < lastWriteMicros);+ assertTrue(lastWriteMicros < ChronoUnit.MICROS.between(Instant.EPOCH, to));+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);+ JsonNode summary = mapper.readTree(files.get("inc/AWSDynamoDB/" + exportId + "/manifest-summary.json"));+ List<String> summaryFields = new ArrayList<>();+ summary.fieldNames().forEachRemaining(summaryFields::add);+ assertEquals(List.of("version", "exportArn", "startTime", "endTime", "tableArn", "tableId", "exportFromTime",+ "exportToTime", "s3Bucket", "s3Prefix", "s3SseAlgorithm", "s3SseKmsKeyId", "manifestFilesS3Key",+ "billedSizeBytes", "itemCount", "outputFormat", "outputView", "exportType"), summaryFields);+ assertEquals("2023-08-01", summary.get("version").asText());+ assertEquals("NEW_AND_OLD_IMAGES", summary.get("outputView").asText());+ assertEquals("INCREMENTAL_EXPORT", summary.get("exportType").asText());+ }++ @Test+ void incrementalIonExportOfNewImagesLeavesOutOldImages() throws Exception {+ S3Service s3 = exportBucket();+ DynamoDbService svc = incrementalExportService(s3);+ svc.putItem("Users", item("userId", "updated", "v", "before"), "us-east-1");+ svc.putItem("Users", item("userId", "deleted", "v", "before"), "us-east-1");+ clock.advance(Duration.ofSeconds(1));+ Instant from = clock.instant();+ svc.putItem("Users", item("userId", "updated", "v", "after"), "us-east-1");+ svc.deleteItem("Users", item("userId", "deleted"), "us-east-1");+ clock.advance(Duration.ofMinutes(20));++ String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "ExportFormat", "ION",+ "ExportType", "INCREMENTAL_EXPORT"), null,+ new DynamoDbService.IncrementalExportRequest(from, null, "NEW_IMAGE"), "us-east-1").getExportArn();++ ExportDescription described = svc.describeExport(exportArn);+ assertEquals("COMPLETED", described.getExportStatus());+ assertEquals(described.getStartTime(), described.getIncrementalExportSpecification().getExportToTime());+ assertEquals(0, described.getStartTime().remainder(BigDecimal.ONE).signum());+ Map<String, byte[]> files = uploadedFiles(s3);+ String dataKey = files.keySet().stream().filter(key -> key.endsWith(".ion.gz")).findFirst().orElseThrow();+ assertTrue(dataKey.matches("AWSDynamoDB/data/[a-z2-7]{26}\\.ion\\.gz"), dataKey);+ assertEquals("""+ $ion_1_0 {Record:{Keys:{userId:"updated"},Metadata:{WriteTimestampMicros:T},NewImage:{userId:"updated",v:"after"}}}+ $ion_1_0 {Record:{Keys:{userId:"deleted"},Metadata:{WriteTimestampMicros:T}}}+ """, gunzip(files.get(dataKey)).replaceAll("WriteTimestampMicros:\\d+d\\d+", "WriteTimestampMicros:T"));+ }++ @Test+ void incrementalExportChecksThePeriodInTheAwsOrder() {+ DynamoDbService svc = incrementalExportService(exportBucket());+ svc.createTable("NoBackups", List.of(new KeySchemaElement("userId", "HASH")),+ List.of(new AttributeDefinition("userId", "S")), 5L, 5L, "us-east-1");+ Instant creationSecond = svc.describeTable("NoBackups", "us-east-1").getCreationDateTime()+ .truncatedTo(ChronoUnit.SECONDS);+ clock.advance(Duration.ofHours(25));+ Instant now = clock.instant();++ Instant beforeCreation = creationSecond.minusMillis(1);+ expectInvalidExportTime(svc, "NoBackups", beforeCreation, now.plusSeconds(60),+ "Incremental export period from time cannot be less than the table creation time : " + beforeCreation);+ Instant afterNow = now.plusSeconds(60);+ expectInvalidExportTime(svc, "NoBackups", afterNow.plusSeconds(60), afterNow,+ "Incremental export period to time cannot be greater than the current time : " + afterNow);+ expectInvalidExportTime(svc, "NoBackups", creationSecond, creationSecond,+ "Incremental export period from time should not be greater or equal to incremental export period to time");+ expectInvalidExportTime(svc, "NoBackups", creationSecond, creationSecond.plus(Duration.ofMinutes(15)).minusMillis(1),+ "Difference between export period from time and export period to time is less than 15 minutes");+ Instant overADay = creationSecond.plus(Duration.ofHours(24)).plusSeconds(1);+ String overADayMessage = "Difference between export period from time and export period to time is greater than 24 hours";+ expectInvalidExportTime(svc, "NoBackups", creationSecond, overADay, overADayMessage);+ for (Duration period : List.of(Duration.ofMinutes(15), Duration.ofHours(24).plusMillis(1))) {+ AwsException noBackups = assertThrows(AwsException.class, () -> incrementalExport(svc, "NoBackups",+ creationSecond, creationSecond.plus(period)));+ assertEquals("PointInTimeRecoveryUnavailableException", noBackups.getErrorCode());+ }+ svc.updateContinuousBackups("Users", false, null, "us-east-1");+ svc.updateContinuousBackups("Users", true, null, "us-east-1");+ expectInvalidExportTime(svc, "Users", creationSecond, overADay, overADayMessage);+ expectInvalidExportTime(svc, "Users", creationSecond, creationSecond.plus(Duration.ofMinutes(15)),+ "Export period from time cannot be earlier than the Point in Time Recovery window");+ }++ private static void expectInvalidExportTime(DynamoDbService svc, String table, Instant from, Instant to,+ String message) {+ AwsException e = assertThrows(AwsException.class, () -> incrementalExport(svc, table, from, to));+ assertEquals("InvalidExportTimeException", e.getErrorCode());+ assertEquals(message, e.getMessage());+ }++ private static ExportDescription incrementalExport(DynamoDbService svc, String table, Instant from, Instant to) {+ return svc.exportTable(Map.of("TableArn", "arn:aws:dynamodb:us-east-1:000000000000:table/" + table,+ "S3Bucket", "bucket", "ExportType", "INCREMENTAL_EXPORT"), null,+ new DynamoDbService.IncrementalExportRequest(from, to, null), "us-east-1");+ }++ /** Every file the export uploaded, by key. */+ private static Map<String, byte[]> uploadedFiles(S3Service s3) {ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);ArgumentCaptor<byte[]> parts = ArgumentCaptor.forClass(byte[].class);verify(s3, atLeastOnce()).uploadPart(eq("bucket"), keys.capture(), anyString(), eq(1), parts.capture());- String dir = "exports-ion/AWSDynamoDB/" + exportArn.substring(exportArn.lastIndexOf('/') + 1) + "/";- List<String> uploadedKeys = keys.getAllValues();- String dataKey = uploadedKeys.stream().filter(key -> key.endsWith(".ion.gz")).findFirst().orElseThrow();- assertTrue(dataKey.matches(dir + "data/[a-z2-7]{26}\\.ion\\.gz"), dataKey);- byte[] dataFile = parts.getAllValues().get(uploadedKeys.indexOf(dataKey));- try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(dataFile))) {- assertEquals("$ion_1_0 {Item:{userId:\"u1\",total:15d1}}\n", new String(gzip.readAllBytes(), StandardCharsets.UTF_8));+ Map<String, byte[]> files = new LinkedHashMap<>();+ for (int i = 0; i < keys.getAllValues().size(); i++) {+ files.put(keys.getAllValues().get(i), parts.getAllValues().get(i));+ }+ return files;+ }++ private static String gunzip(byte[] data) throws IOException {+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data))) {+ return new String(gzip.readAllBytes(), StandardCharsets.UTF_8);}- byte[] summary = parts.getAllValues().get(uploadedKeys.indexOf(dir + "manifest-summary.json"));- assertEquals("ION", mapper.readTree(summary).get("outputFormat").asText());}/** A bucket that takes every upload, each completing with a one-part ETag. */@@ -4599,6 +4777,12 @@ class DynamoDbServiceTest {return svc;}+ /** Tables take their creation time from the system clock, so this clock starts after it. */+ private DynamoDbService incrementalExportService(S3Service s3) {+ clock.advance(Duration.between(clock.instant(), Instant.now()).plusSeconds(1));+ return inlineExportService(s3);+ }+@Testvoid validateImportRequest_rejectsUnsupportedFormatWithoutUnsupportedOperationWording() {AwsException e = assertThrows(AwsException.class,