Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): export the table as of ExportTime (#6132)
TATSUNO “Taz” Yasuhiroda66550c702a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.javaindex e6d9929ef5..5c1d6b461d 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java@@ -12,6 +12,8 @@ import java.io.BufferedReader;import java.io.ByteArrayInputStream;import java.io.InputStreamReader;import java.nio.charset.StandardCharsets;+import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.List;import java.util.Map;import java.util.zip.GZIPInputStream;@@ -53,6 +55,11 @@ class DynamoDbExportTest {.billingMode(BillingMode.PAY_PER_REQUEST).build());tableArn = tableResp.tableDescription().tableArn();+ ddb.updateContinuousBackups(UpdateContinuousBackupsRequest.builder()+ .tableName(TABLE_NAME)+ .pointInTimeRecoverySpecification(PointInTimeRecoverySpecification.builder()+ .pointInTimeRecoveryEnabled(true).build())+ .build());// Insert 3 itemsddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)@@ -224,6 +231,46 @@ class DynamoDbExportTest {.isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class);}+ @Test+ @Order(8)+ void exportTableToPointInTime_exportTime_exportsTheItemsOfThatTime() throws InterruptedException {+ Thread.sleep(50);+ Instant beforeFourthItem = Instant.now();+ Thread.sleep(50);+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)+ .item(Map.of(+ "pk", AttributeValue.fromS("user-4"),+ "sk", AttributeValue.fromS("order-004"),+ "total", AttributeValue.fromN("10")))+ .build());++ ExportDescription desc = ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .s3Prefix("exports-time")+ .exportTime(beforeFourthItem)+ .build()).exportDescription();+ assertThat(desc.exportTime()).isEqualTo(beforeFourthItem.truncatedTo(ChronoUnit.MILLIS));++ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {+ waiter.waitUntilExportCompleted(r -> r.exportArn(desc.exportArn()));+ }+ assertThat(ddb.describeExport(DescribeExportRequest.builder().exportArn(desc.exportArn()).build())+ .exportDescription().itemCount()).isEqualTo(3L);+ }++ @Test+ @Order(9)+ void exportTableToPointInTime_futureExportTime_throwsInvalidExportTimeException() {+ assertThatThrownBy(() -> ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .exportTime(Instant.now().plusSeconds(3600))+ .build()))+ .isInstanceOf(InvalidExportTimeException.class)+ .hasMessageContaining("Export Time is invalid");+ }+private String decompressGzip(byte[] data) throws Exception {try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 110b8c7d8d..6d023d42bf 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -393,12 +393,16 @@ Deviations from AWS:## Export to S3-Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format):+Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format). As on AWS, the table needs point-in-time recovery:```bash# Create a bucket to receive the exportaws s3 mb s3://my-exports --endpoint-url $AWS_ENDPOINT_URL+# Turn on point-in-time recovery+aws dynamodb update-continuous-backups --table-name Users \+ --point-in-time-recovery-specification PointInTimeRecoveryEnabled=true --endpoint-url $AWS_ENDPOINT_URL+# Start an exportEXPORT_ARN=$(aws dynamodb export-table-to-point-in-time \--table-arn arn:aws:dynamodb:us-east-1:000000000000:table/Users \@@ -422,6 +426,8 @@ aws dynamodb list-exports \The export writes to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/data/` as one or more `.json.gz` files, along with `manifest-summary.json` and `manifest-files.json`, the same layout as real AWS DynamoDB exports.+The export holds the items as they were at `ExportTime`, or at the request time when `ExportTime` is not set. As on AWS, `ExportTime` must not be before `EarliestRestorableDateTime` or the table creation, and must not be in the future. A time after `LatestRestorableDateTime` is accepted.+## Import from S3Create a new table and load it from newline-delimited DynamoDB JSON objects in S3. Each line is `{"Item": {...}}`, the format an export writes:@@ -486,7 +492,7 @@ aws dynamodb restore-table-to-point-in-time --source-table-name Users \Deleting a table with point-in-time recovery enabled leaves a `SYSTEM` backup named `<table>$DeletedTableBackup`. It holds the table as it was at deletion, cannot be deleted, and expires after 35 days. `ListBackups` shows it with `BackupType` `SYSTEM` or `ALL`, and `RestoreTableFromBackup` restores it.-Deviations from AWS: enabling point-in-time recovery right after `CreateTable` is accepted at once. AWS sometimes refuses it for a few seconds with `ContinuousBackupsUnavailableException`.+Deviations from AWS: enabling point-in-time recovery right after `CreateTable` is accepted at once. AWS sometimes refuses it for a few seconds with `ContinuousBackupsUnavailableException`. `EarliestRestorableDateTime` is rounded down to the second, as on AWS. A restore or an export at that time on AWS leaves out the writes made earlier in that second, before point-in-time recovery was enabled. Floci keeps changes only from the moment it is enabled, so it includes those writes.## Kinesis change data capture (CDC)diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 2f2f4de9e2..d0f77cdd7d 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -80,6 +80,9 @@ import java.util.stream.Collectors;import java.util.zip.GZIPInputStream;import java.util.zip.GZIPOutputStream;+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.epochSeconds;+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.instantOf;+@ApplicationScopedpublic class DynamoDbService {@@ -4718,7 +4721,8 @@ public class DynamoDbService {// --- Export Operations ---- public ExportDescription exportTable(Map<String, Object> request, String region) {+ /** As on AWS, an ExportTime after LatestRestorableDateTime is accepted up to now. */+ public ExportDescription exportTable(Map<String, Object> request, Instant exportTime, String region) {String tableArn = (String) request.get("TableArn");String s3Bucket = (String) request.get("S3Bucket");String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;@@ -4742,10 +4746,24 @@ public class DynamoDbService {String tableRegion = ref.region() != null ? ref.region() : region;String storageKey = regionKey(tableRegion, tableName);- TableDefinition table = tableStore.get(storageKey)- .orElseThrow(() -> resourceNotFoundException(tableName));+ TableDefinition table = requireBackupSource(storageKey, tableArn);+ Instant startTime = clock.instant().truncatedTo(ChronoUnit.MILLIS);+ // Seconds, as CreationDateTime is shown, so an EarliestRestorableDateTime in the creation second passes.+ if (exportTime != null && (exportTime.isBefore(table.getCreationDateTime().truncatedTo(ChronoUnit.SECONDS))+ || startTime.isBefore(exportTime))) {+ throw new AwsException("InvalidExportTimeException", "Export Time is invalid", 400);+ }+ if (!table.isPointInTimeRecoveryEnabled()) {+ throw new AwsException("PointInTimeRecoveryUnavailableException",+ "Point in time recovery is not enabled for table '" + table.getTableName() + "'", 400);+ }+ if (exportTime != null+ && exportTime.isBefore(pointInTimeRecovery.window(withPointInTimeStart(storageKey, table)).earliest())) {+ throw new AwsException("InvalidExportTimeException",+ "Point in Time Recovery is not available for specified export time", 400);+ }+ Instant time = exportTime != null ? exportTime : startTime;- long now = Instant.now().getEpochSecond();var exportId = newJobId();String exportArn = AwsArnUtils.Arn.of("dynamodb", tableRegion, regionResolver.getAccountId(), "table/" + table.getTableName() + "/export/" + exportId).toString();@@ -4758,8 +4776,8 @@ public class DynamoDbService {desc.setS3Prefix(s3Prefix);desc.setExportFormat(exportFormat);desc.setExportType("FULL_EXPORT");- desc.setExportTime(now);- desc.setStartTime(now);+ desc.setExportTime(epochSeconds(time));+ desc.setStartTime(epochSeconds(startTime));desc.setClientToken(clientToken);desc.setS3SseAlgorithm(s3SseAlgorithm);desc.setS3BucketOwner(s3BucketOwner);@@ -4769,9 +4787,8 @@ public class DynamoDbService {}ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(scopedItemsKey(storageKey));- List<JsonNode> snapshot = tableItems != null- ? List.copyOf(tableItems.values())- : List.of();+ List<JsonNode> snapshot = pointInTimeRecovery.itemsAt(regionResolver.getAccountId(), storageKey,+ tableItems != null ? tableItems : Map.of(), time);// The worker gets its own copy so the object serialized into this response never// changes underneath the handler, and the request account so its writes land there.@@ -4804,7 +4821,7 @@ public class DynamoDbService {desc.setExportStatus("FAILED");desc.setFailureCode("S3NoSuchBucket");desc.setFailureMessage("The specified bucket does not exist: " + s3Bucket);- desc.setEndTime(Instant.now().getEpochSecond());+ desc.setEndTime(epochSeconds(clock.instant()));if (exportStore != null) {exportStore.put(exportArn, desc);}@@ -4830,9 +4847,8 @@ public class DynamoDbService {manifestSummaryContent.getBytes(StandardCharsets.UTF_8),"application/json", Map.of());- long endTime = Instant.now().getEpochSecond();desc.setExportStatus("COMPLETED");- desc.setEndTime(endTime);+ desc.setEndTime(epochSeconds(clock.instant()));desc.setItemCount(itemCount);desc.setBilledSizeBytes(billedSize);desc.setExportManifest(manifestSummaryKey);@@ -4847,7 +4863,7 @@ public class DynamoDbService {desc.setExportStatus("FAILED");desc.setFailureCode("UNKNOWN");desc.setFailureMessage(e.getMessage());- desc.setEndTime(Instant.now().getEpochSecond());+ desc.setEndTime(epochSeconds(clock.instant()));if (exportStore != null) {exportStore.put(exportArn, desc);}@@ -4885,11 +4901,11 @@ public class DynamoDbService {ObjectNode root = objectMapper.createObjectNode();root.put("version", "2020-06-30");root.put("exportArn", desc.getExportArn());- root.put("startTime", Instant.ofEpochSecond(desc.getStartTime()).toString());+ root.put("startTime", instantOf(desc.getStartTime()).toString());root.put("endTime", Instant.now().toString());root.put("tableArn", desc.getTableArn());root.put("tableId", desc.getTableId());- root.put("exportTime", Instant.ofEpochSecond(desc.getExportTime()).toString());+ root.put("exportTime", instantOf(desc.getExportTime()).toString());root.put("s3Bucket", desc.getS3Bucket());root.putNull("s3Prefix");if (desc.getS3Prefix() != null) {@@ -5379,7 +5395,7 @@ public class DynamoDbService {desc.setExportStatus("FAILED");desc.setFailureCode("InterruptedByRestart");desc.setFailureMessage("The emulator restarted before the export finished");- desc.setEndTime(now);+ desc.setEndTime(BigDecimal.valueOf(now));exports.putForAccount(entry.accountId(), entry.key(), desc);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.javanew file mode 100644index 0000000000..a08744736b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.java@@ -0,0 +1,17 @@+package io.github.hectorvent.floci.services.dynamodb;++import java.math.BigDecimal;+import java.time.Instant;++final class DynamoDbTimestamps {++ private DynamoDbTimestamps() {}++ static BigDecimal epochSeconds(Instant instant) {+ return BigDecimal.valueOf(instant.toEpochMilli(), 3);+ }++ static Instant instantOf(BigDecimal epochSeconds) {+ return Instant.ofEpochMilli(epochSeconds.movePointRight(3).longValue());+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex 46cbff0310..ac18b2a578 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -33,7 +33,6 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import jakarta.ws.rs.core.Response;-import java.math.BigDecimal;import java.time.Instant;import java.util.ArrayList;import java.util.Collections;@@ -48,6 +47,8 @@ import java.util.Set;import java.util.function.Supplier;import java.util.regex.Pattern;+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.epochSeconds;+/*** DynamoDB JSON protocol handler of the native engine, reached through the native backend* behind the DynamoDB operation seam rather than directly from the controllers.@@ -60,6 +61,9 @@ public class NativeDynamoDbJsonHandler {// The order AWS prints the enum constraint in.private static final List<String> BACKUP_TYPE_FILTERS = List.of("ALL", "AWS_BACKUP", "USER", "SYSTEM");private static final int MAX_LIST_BACKUPS_LIMIT = 100;+ private static final int MIN_BACKUP_ARN_LENGTH = 37;+ private static final int MAX_BACKUP_ARN_LENGTH = 1024;+ private static final Instant MAX_TIME_RANGE_BOUND = Instant.ofEpochMilli(99_999_999_999_999L);private static final int MAX_RECOVERY_PERIOD_DAYS = 35;private final DynamoDbService dynamoDbService;@@ -1961,6 +1965,15 @@ public class NativeDynamoDbJsonHandler {violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "+ "Member must have value less than or equal to " + MAX_LIST_BACKUPS_LIMIT);}+ String exclusiveStartBackupArn = textOrNull(request, "ExclusiveStartBackupArn");+ if (exclusiveStartBackupArn != null && exclusiveStartBackupArn.length() < MIN_BACKUP_ARN_LENGTH) {+ violations.add("Value '" + exclusiveStartBackupArn + "' at 'exclusiveStartBackupArn' failed to satisfy "+ + "constraint: Member must have length greater than or equal to " + MIN_BACKUP_ARN_LENGTH);+ }+ if (exclusiveStartBackupArn != null && MAX_BACKUP_ARN_LENGTH < exclusiveStartBackupArn.length()) {+ violations.add("Value '" + exclusiveStartBackupArn + "' at 'exclusiveStartBackupArn' failed to satisfy "+ + "constraint: Member must have length less than or equal to " + MAX_BACKUP_ARN_LENGTH);+ }String backupType = request.hasNonNull("BackupType") ? request.get("BackupType").asText() : "USER";if (!BACKUP_TYPE_FILTERS.contains(backupType)) {violations.add("Value '" + backupType + "' at 'backupType' failed to satisfy constraint: "@@ -1973,9 +1986,11 @@ public class NativeDynamoDbJsonHandler {throw new AwsException("ValidationException", "One or more parameter values were invalid: "+ "Time range lower bound must be less than or equal to upper bound", 400);}+ checkTimeRangeBound("TimeRangeLowerBound", lowerBound);+ checkTimeRangeBound("TimeRangeUpperBound", upperBound);DynamoDbService.ListBackupsResult result = dynamoDbService.listBackups(tableName, limit, lowerBound,- upperBound, textOrNull(request, "ExclusiveStartBackupArn"), backupType, region);+ upperBound, exclusiveStartBackupArn, backupType, region);ObjectNode response = objectMapper.createObjectNode();ArrayNode summaries = response.putArray("BackupSummaries");@@ -2211,13 +2226,21 @@ public class NativeDynamoDbJsonHandler {return node.hasNonNull(member) ? node.get(member).asText() : null;}- private static Instant timestampOrNull(JsonNode request, String member) {- JsonNode node = request.path(member);- return node.isNumber() ? Instant.ofEpochMilli(node.decimalValue().movePointRight(3).longValue()) : null;+ private static void checkTimeRangeBound(String member, Instant bound) {+ if (bound != null && bound.isBefore(Instant.EPOCH)) {+ throw new AwsException("ValidationException",+ "Invalid Request: The supplied " + member + " must not be before " + Instant.EPOCH, 400);+ }+ if (bound != null && MAX_TIME_RANGE_BOUND.isBefore(bound)) {+ throw new AwsException("ValidationException",+ "Invalid Request: The supplied " + member + " must not be after " + MAX_TIME_RANGE_BOUND, 400);+ }}- private static BigDecimal epochSeconds(Instant instant) {- return BigDecimal.valueOf(instant.toEpochMilli(), 3);+ private static Instant timestampOrNull(JsonNode request, String member) {+ JsonNode node = request.path(member);+ // AWS rounds the seconds as a double, so a value beyond the long range of milliseconds saturates.+ return node.isNumber() ? Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000)) : null;}private ObjectNode backupDetailsNode(TableBackup backup, String status) {@@ -3384,7 +3407,7 @@ public class NativeDynamoDbJsonHandler {? e.getValue().asText() : e.getValue()));io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =- dynamoDbService.exportTable(params, region);+ dynamoDbService.exportTable(params, timestampOrNull(request, "ExportTime"), region);ObjectNode response = objectMapper.createObjectNode();response.set("ExportDescription", objectMapper.valueToTree(desc));diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.javaindex 4ae924b18d..ceed3f3df1 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java@@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonProperty;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.math.BigDecimal;+@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@JsonInclude(JsonInclude.Include.NON_NULL)@@ -35,13 +37,13 @@ public class ExportDescription {private String exportType;@JsonProperty("ExportTime")- private Long exportTime;+ private BigDecimal exportTime;@JsonProperty("StartTime")- private Long startTime;+ private BigDecimal startTime;@JsonProperty("EndTime")- private Long endTime;+ private BigDecimal endTime;@JsonProperty("ItemCount")private Long itemCount;@@ -91,14 +93,14 @@ public class ExportDescription {public String getExportType() { return exportType; }public void setExportType(String exportType) { this.exportType = exportType; }- public Long getExportTime() { return exportTime; }- public void setExportTime(Long exportTime) { this.exportTime = exportTime; }+ public BigDecimal getExportTime() { return exportTime; }+ public void setExportTime(BigDecimal exportTime) { this.exportTime = exportTime; }- public Long getStartTime() { return startTime; }- public void setStartTime(Long startTime) { this.startTime = startTime; }+ public BigDecimal getStartTime() { return startTime; }+ public void setStartTime(BigDecimal startTime) { this.startTime = startTime; }- public Long getEndTime() { return endTime; }- public void setEndTime(Long endTime) { this.endTime = endTime; }+ public BigDecimal getEndTime() { return endTime; }+ public void setEndTime(BigDecimal endTime) { this.endTime = endTime; }public Long getItemCount() { return itemCount; }public void setItemCount(Long itemCount) { this.itemCount = itemCount; }diff --git a/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.javaindex c7921232c3..30bde37e0b 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java@@ -102,6 +102,13 @@ class RegionScopedListsIntegrationTest {""".formatted(table)).when().post("/").then().statusCode(200).extract().path("TableDescription.TableArn");+ given().header("Authorization", auth(HOME, "dynamodb"))+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateContinuousBackups")+ .contentType(DYNAMODB_JSON)+ .body("""+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}+ """.formatted(table))+ .when().post("/").then().statusCode(200);// The bucket is never created: the export job is recorded before it writes anything, and a// missing bucket only fails the job later. Export jobs have no delete API.String exportArn = given().header("Authorization", auth(HOME, "dynamodb"))diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.javaindex 8d9300ea8d..417f188b15 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java@@ -269,6 +269,33 @@ class DynamoDbBackupIntegrationTest {+ "Time range lower bound must be less than or equal to upper bound"));}+ @Test+ void listBackupsChecksTheTimeRangeLimits() {+ String before = "Invalid Request: The supplied %s must not be before 1970-01-01T00:00:00Z";+ String after = "Invalid Request: The supplied %s must not be after 5138-11-16T09:46:39.999Z";++ listBackupsBetween("-0.0005", "99999999999.9994").statusCode(200);+ listBackupsBetween("-0.001", null)+ .statusCode(400)+ .body("message", equalTo(before.formatted("TimeRangeLowerBound")));+ listBackupsBetween(null, "100000000000")+ .statusCode(400)+ .body("message", equalTo(after.formatted("TimeRangeUpperBound")));+ listBackupsBetween("-1e20", "1e20")+ .statusCode(400)+ .body("message", equalTo(before.formatted("TimeRangeLowerBound")));+ listBackupsBetween("0", "1e400")+ .statusCode(400)+ .body("message", equalTo(after.formatted("TimeRangeUpperBound")));+ listBackupsBetween("1e21", "1e20")+ .statusCode(400)+ .body("message", equalTo(after.formatted("TimeRangeLowerBound")));+ listBackupsBetween("100000000000", "-0.001")+ .statusCode(400)+ .body("message", equalTo("One or more parameter values were invalid: "+ + "Time range lower bound must be less than or equal to upper bound"));+ }+@Testvoid listBackupsReportsEveryViolation() {ddb("ListBackups", "{\"Limit\": 0, \"BackupType\": \"BAD\"}")@@ -284,6 +311,22 @@ class DynamoDbBackupIntegrationTest {ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + "x".repeat(40) + "\"}").statusCode(400).body("message", equalTo("Invalid Backup ARN"));+ String shortArn = "x".repeat(36);+ ddb("ListBackups", """+ {"Limit": 0, "ExclusiveStartBackupArn": "%s", "BackupType": "BAD"}+ """.formatted(shortArn))+ .statusCode(400)+ .body("message", equalTo("3 validation errors detected: Value '0' at 'limit' failed to satisfy "+ + "constraint: Member must have value greater than or equal to 1; Value '" + shortArn+ + "' at 'exclusiveStartBackupArn' failed to satisfy constraint: Member must have length "+ + "greater than or equal to 37; Value 'BAD' at 'backupType' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [ALL, AWS_BACKUP, USER, SYSTEM]"));+ String longArn = "x".repeat(1025);+ ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + longArn + "\"}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '" + longArn + "' at "+ + "'exclusiveStartBackupArn' failed to satisfy constraint: Member must have length "+ + "less than or equal to 1024"));}@Test@@ -755,4 +798,10 @@ class DynamoDbBackupIntegrationTest {{"BackupArn": "%s", "TargetTableName": "%s", %s}""".formatted(backupArn, target, overrides));}++ private static ValidatableResponse listBackupsBetween(String lowerBound, String upperBound) {+ return ddb("ListBackups", """+ {"TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}+ """.formatted(lowerBound, upperBound));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.javaindex e8d45128ba..4c95f667c6 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.dynamodb;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.path.json.JsonPath;+import io.restassured.response.ValidatableResponse;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -9,9 +11,15 @@ import org.junit.jupiter.api.Test;import java.io.BufferedReader;import java.io.ByteArrayInputStream;import java.io.InputStreamReader;+import java.math.BigDecimal;+import java.math.BigInteger;import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.List;import java.util.zip.GZIPInputStream;+import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;+import static io.github.hectorvent.floci.testing.DynamoDbRequests.decimalAt;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;import static org.junit.jupiter.api.Assertions.*;@@ -24,6 +32,7 @@ class DynamoDbExportIntegrationTest {private static final String BUCKET_NAME = "export-test-bucket";private static final String TABLE_ARN ="arn:aws:dynamodb:us-east-1:000000000000:table/" + TABLE_NAME;+ private static final String TABLE_ARN_PREFIX = "arn:aws:dynamodb:us-east-1:000000000000:table/";private static boolean setupDone = false;@@ -59,6 +68,7 @@ class DynamoDbExportIntegrationTest {}""".formatted(TABLE_NAME)).when().post("/").then().statusCode(200);+ enablePointInTimeRecovery(TABLE_NAME);// Insert 3 itemsputItem("{\"pk\": {\"S\": \"user-1\"}, \"sk\": {\"S\": \"order-001\"}, \"total\": {\"N\": \"99\"}}");@@ -175,7 +185,7 @@ class DynamoDbExportIntegrationTest {}@Test- void exportTableToPointInTime_tableNotFound_returnsResourceNotFoundException() {+ void exportTableToPointInTime_tableNotFound_returnsTableNotFoundException() {given().header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime").contentType(DYNAMODB_CONTENT_TYPE)@@ -188,7 +198,9 @@ class DynamoDbExportIntegrationTest {.when().post("/").then().statusCode(400)- .body("__type", containsString("ResourceNotFoundException"));+ .body("__type", containsString("TableNotFoundException"))+ .body("message", equalTo(+ "Table not found: arn:aws:dynamodb:us-east-1:000000000000:table/NonExistentTable"));}@Test@@ -249,8 +261,145 @@ class DynamoDbExportIntegrationTest {}}+ @Test+ void exportTimeExportsTheItemsOfThatTime() throws Exception {+ String table = "ExportTimeItems";+ createHashTable(table);+ BigDecimal earliest = decimalAt(enablePointInTimeRecovery(table),+ "/ContinuousBackupsDescription/PointInTimeRecoveryDescription/EarliestRestorableDateTime");+ putHashItem(table, "k1", "v1");+ BigDecimal t1 = pause();+ putHashItem(table, "k1", "v2");+ putHashItem(table, "k2", "v1");++ exportAt(table, "export-time-earliest", earliest).statusCode(200);+ ValidatableResponse atT1 = exportAt(table, "export-time-t1", t1).statusCode(200);+ assertEquals(0, t1.compareTo(decimalAt(atT1, "/ExportDescription/ExportTime")));+ long before = System.currentTimeMillis();+ ValidatableResponse noTime = ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "export-time-now"}+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)).statusCode(200);+ long after = System.currentTimeMillis();+ BigDecimal startTime = decimalAt(noTime, "/ExportDescription/StartTime");+ assertEquals(0, startTime.compareTo(decimalAt(noTime, "/ExportDescription/ExportTime")));+ long startMillis = startTime.movePointRight(3).longValueExact();+ assertTrue(before <= startMillis && startMillis <= after, before + " <= " + startTime + " <= " + after);++ String atT1Arn = atT1.extract().path("ExportDescription.ExportArn");+ assertEquals("COMPLETED", pollUntilCompleted(atT1Arn, 10_000));+ assertEquals(List.of("k1=v1"), exportedItems(atT1Arn, "export-time-t1"));+ String noTimeArn = noTime.extract().path("ExportDescription.ExportArn");+ assertEquals("COMPLETED", pollUntilCompleted(noTimeArn, 10_000));+ assertEquals(List.of("k1=v2", "k2=v1"), exportedItems(noTimeArn, "export-time-now"));+ }++ @Test+ void exportTableToPointInTimeChecksInTheAwsOrder() throws Exception {+ String table = "ExportTimeChecks";+ long beforeCreate = System.currentTimeMillis();+ createHashTable(table);+ long afterCreate = System.currentTimeMillis();+ BigDecimal future = BigDecimal.valueOf(System.currentTimeMillis() + 60_000, 3);++ expectError(exportAt("ExportTimeMissing", "export-checks", future),+ "TableNotFoundException", "Table not found: " + TABLE_ARN_PREFIX + "ExportTimeMissing");+ expectError(exportAt(table, "export-checks", future), "InvalidExportTimeException", "Export Time is invalid");+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s"}+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)), "PointInTimeRecoveryUnavailableException",+ "Point in time recovery is not enabled for table '" + table + "'");++ // EarliestRestorableDateTime is in whole seconds. Enabling in a later second puts it after the creation.+ while (System.currentTimeMillis() / 1000 <= afterCreate / 1000) {+ Thread.sleep(10);+ }+ BigDecimal earliest = decimalAt(enablePointInTimeRecovery(table),+ "/ContinuousBackupsDescription/PointInTimeRecoveryDescription/EarliestRestorableDateTime");+ expectError(exportAt(table, "export-checks", BigDecimal.valueOf(beforeCreate - 1000, 3)),+ "InvalidExportTimeException", "Export Time is invalid");+ expectError(exportAt(table, "export-checks", earliest.subtract(new BigDecimal("0.001"))),+ "InvalidExportTimeException", "Point in Time Recovery is not available for specified export time");+ expectError(exportAt(table, "export-checks", future), "InvalidExportTimeException", "Export Time is invalid");+ exportAt(table, "export-checks", earliest).statusCode(200);+ }++ @Test+ void exportTimeBeyondTheMillisecondRangeIsInvalid() {+ String table = "ExportTimeRange";+ createHashTable(table);+ enablePointInTimeRecovery(table);+ BigDecimal nowPlusTwoToThe64Millis = BigDecimal.valueOf(System.currentTimeMillis(), 3)+ .add(new BigDecimal(BigInteger.TWO.pow(64), 3));++ expectError(exportAt(table, "export-range", nowPlusTwoToThe64Millis),+ "InvalidExportTimeException", "Export Time is invalid");+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "ExportTime": 1e400}+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)),+ "InvalidExportTimeException", "Export Time is invalid");+ }+// --- Helpers ---+ /** Returns a time between the writes before and after the call, in epoch seconds with milliseconds. */+ private static BigDecimal pause() {+ long now = System.currentTimeMillis();+ while (System.currentTimeMillis() <= now) {+ Thread.onSpinWait();+ }+ return BigDecimal.valueOf(now, 3);+ }++ private static void createHashTable(String table) {+ ddb("CreateTable", """+ {"TableName": "%s", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],+ "BillingMode": "PAY_PER_REQUEST"}+ """.formatted(table)).statusCode(200);+ }++ private static void putHashItem(String table, String pk, String value) {+ ddb("PutItem", """+ {"TableName": "%s", "Item": {"pk": {"S": "%s"}, "v": {"S": "%s"}}}+ """.formatted(table, pk, value)).statusCode(200);+ }++ private static ValidatableResponse exportAt(String table, String prefix, BigDecimal exportTime) {+ return ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "%s", "ExportTime": %s}+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME, prefix, exportTime.toPlainString()));+ }++ private List<String> exportedItems(String exportArn, String prefix) throws Exception {+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);+ String dataKey = given()+ .when().get("/" + BUCKET_NAME + "/" + prefix + "/AWSDynamoDB/" + exportId + "/manifest-files.json")+ .then().statusCode(200)+ .extract().asString().trim();+ byte[] gzipData = given()+ .when().get("/" + BUCKET_NAME + "/" + dataKey)+ .then().statusCode(200)+ .extract().asByteArray();+ List<String> items = new ArrayList<>();+ for (String line : decompressGzip(gzipData).split("\n")) {+ JsonPath item = JsonPath.from(line);+ items.add(item.getString("Item.pk.S") + "=" + item.getString("Item.v.S"));+ }+ return items.stream().sorted().toList();+ }++ private static void expectError(ValidatableResponse response, String type, String message) {+ response.statusCode(400)+ .body("__type", containsString(type))+ .body("message", equalTo(message));+ }++ private static ValidatableResponse enablePointInTimeRecovery(String table) {+ return ddb("UpdateContinuousBackups", """+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}+ """.formatted(table)).statusCode(200);+ }+private void putItem(String itemJson) {given().header("X-Amz-Target", "DynamoDB_20120810.PutItem")diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.javaindex cb198f0da5..d7b796cff5 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java@@ -109,6 +109,9 @@ class DynamoDbImportIntegrationTest {@Testvoid importTable_gzip_roundTripsAnExport() {createCompositeTable("RoundTripSource");+ dynamo("UpdateContinuousBackups", """+ {"TableName": "RoundTripSource", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}+ """).then().statusCode(200);for (int i = 1; i <= 3; i++) {dynamo("PutItem", "{\"TableName\": \"RoundTripSource\", \"Item\": {\"pk\": {\"S\": \"user-" + i+ "\"}, \"sk\": {\"S\": \"order-" + i + "\"}, \"total\": {\"N\": \"" + (i * 10) + "\"}}}")diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.javaindex 8368ad6398..37ee8d506f 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java@@ -7,6 +7,7 @@ import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;import java.math.BigDecimal;+import java.math.BigInteger;import java.time.Duration;import java.time.Instant;import java.util.List;@@ -186,6 +187,22 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {restoreToTime(table, "PitrIT-window-checks-earliest", earliest).statusCode(200);}+ @Test+ void restoreDateTimeIsRoundedToTheMillisecondAsADouble() {+ String table = "PitrIT-rounding";+ createTable(table);+ updateContinuousBackups(table, "true").statusCode(200);+ String nowPlusTwoToThe64Millis = BigDecimal.valueOf(System.currentTimeMillis(), 3)+ .add(new BigDecimal(BigInteger.TWO.pow(64), 3)).toPlainString();++ assertEquals("+292278994-08-17T07:12:55.807Z", refusedRestoreTime(table, nowPlusTwoToThe64Millis));+ assertEquals("+292278994-08-17T07:12:55.807Z", refusedRestoreTime(table, "1e400"));+ assertEquals("-292275055-05-16T16:47:04.192Z", refusedRestoreTime(table, "-1e20"));+ assertEquals("1970-01-01T00:00:00.001Z", refusedRestoreTime(table, "0.0005"));+ assertEquals("1970-01-01T00:00:00Z", refusedRestoreTime(table, "-0.0005"));+ assertEquals("2026-10-22T00:42:24.001Z", refusedRestoreTime(table, "1792629744.0004999"));+ }+@Testvoid restoreTableToPointInTimeChecksInTheAwsOrder() {String table = "PitrIT-checks";@@ -425,6 +442,16 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {return ddb("RestoreTableToPointInTime", "{" + members + "}");}+ private String refusedRestoreTime(String source, String restoreDateTime) {+ String message = restore("""+ "SourceTableName": "%s", "TargetTableName": "PitrIT-never", "RestoreDateTime": %s+ """.formatted(source, restoreDateTime))+ .statusCode(400)+ .body("__type", containsString("InvalidRestoreTimeException"))+ .extract().path("message");+ return message.substring("RestoreDateTime '".length(), message.indexOf("' must be within"));+ }+private static void expectError(ValidatableResponse response, String type, String message) {response.statusCode(400).body("__type", containsString(type))diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex ff6229bb8d..809a4fb032 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -4493,6 +4493,19 @@ class DynamoDbServiceTest {assertEquals("ResourceInUseException", e.getErrorCode());}+ @Test+ void exportTable_whileCreating_returnsTableNotFoundException() {+ createUsersTableInCreating(service);+ String tableArn = "arn:aws:dynamodb:us-east-1:000000000000:table/Users";+ Map<String, Object> request = Map.of("TableArn", tableArn, "S3Bucket", "bucket");++ AwsException e = assertThrows(AwsException.class,+ () -> service.exportTable(request, Instant.EPOCH, "us-east-1"));++ assertEquals("TableNotFoundException", e.getErrorCode());+ assertEquals("Table not found: " + tableArn, e.getMessage());+ }+@Testvoid validateImportRequest_rejectsUnsupportedFormatWithoutUnsupportedOperationWording() {AwsException e = assertThrows(AwsException.class,