Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): export the table as of ExportTime (#6132)

TATSUNO “Taz” Yasuhiroda66550c702a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
index e6d9929ef5..5c1d6b461d 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
@@ -12,6 +12,8 @@ import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;
@@ -53,6 +55,11 @@ class DynamoDbExportTest {
.billingMode(BillingMode.PAY_PER_REQUEST)
.build());
tableArn = tableResp.tableDescription().tableArn();
+ ddb.updateContinuousBackups(UpdateContinuousBackupsRequest.builder()
+ .tableName(TABLE_NAME)
+ .pointInTimeRecoverySpecification(PointInTimeRecoverySpecification.builder()
+ .pointInTimeRecoveryEnabled(true).build())
+ .build());
// Insert 3 items
ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
@@ -224,6 +231,46 @@ class DynamoDbExportTest {
.isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class);
}
+ @Test
+ @Order(8)
+ void exportTableToPointInTime_exportTime_exportsTheItemsOfThatTime() throws InterruptedException {
+ Thread.sleep(50);
+ Instant beforeFourthItem = Instant.now();
+ Thread.sleep(50);
+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
+ .item(Map.of(
+ "pk", AttributeValue.fromS("user-4"),
+ "sk", AttributeValue.fromS("order-004"),
+ "total", AttributeValue.fromN("10")))
+ .build());
+
+ ExportDescription desc = ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()
+ .tableArn(tableArn)
+ .s3Bucket(BUCKET_NAME)
+ .s3Prefix("exports-time")
+ .exportTime(beforeFourthItem)
+ .build()).exportDescription();
+ assertThat(desc.exportTime()).isEqualTo(beforeFourthItem.truncatedTo(ChronoUnit.MILLIS));
+
+ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {
+ waiter.waitUntilExportCompleted(r -> r.exportArn(desc.exportArn()));
+ }
+ assertThat(ddb.describeExport(DescribeExportRequest.builder().exportArn(desc.exportArn()).build())
+ .exportDescription().itemCount()).isEqualTo(3L);
+ }
+
+ @Test
+ @Order(9)
+ void exportTableToPointInTime_futureExportTime_throwsInvalidExportTimeException() {
+ assertThatThrownBy(() -> ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()
+ .tableArn(tableArn)
+ .s3Bucket(BUCKET_NAME)
+ .exportTime(Instant.now().plusSeconds(3600))
+ .build()))
+ .isInstanceOf(InvalidExportTimeException.class)
+ .hasMessageContaining("Export Time is invalid");
+ }
+
private String decompressGzip(byte[] data) throws Exception {
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));
BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 110b8c7d8d..6d023d42bf 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -393,12 +393,16 @@ Deviations from AWS:
## Export to S3
-Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format):
+Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format). As on AWS, the table needs point-in-time recovery:
```bash
# Create a bucket to receive the export
aws s3 mb s3://my-exports --endpoint-url $AWS_ENDPOINT_URL
+# Turn on point-in-time recovery
+aws dynamodb update-continuous-backups --table-name Users \
+ --point-in-time-recovery-specification PointInTimeRecoveryEnabled=true --endpoint-url $AWS_ENDPOINT_URL
+
# Start an export
EXPORT_ARN=$(aws dynamodb export-table-to-point-in-time \
--table-arn arn:aws:dynamodb:us-east-1:000000000000:table/Users \
@@ -422,6 +426,8 @@ aws dynamodb list-exports \
The export writes to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/data/` as one or more `.json.gz` files, along with `manifest-summary.json` and `manifest-files.json`, the same layout as real AWS DynamoDB exports.
+The export holds the items as they were at `ExportTime`, or at the request time when `ExportTime` is not set. As on AWS, `ExportTime` must not be before `EarliestRestorableDateTime` or the table creation, and must not be in the future. A time after `LatestRestorableDateTime` is accepted.
+
## Import from S3
Create a new table and load it from newline-delimited DynamoDB JSON objects in S3. Each line is `{"Item": {...}}`, the format an export writes:
@@ -486,7 +492,7 @@ aws dynamodb restore-table-to-point-in-time --source-table-name Users \
Deleting a table with point-in-time recovery enabled leaves a `SYSTEM` backup named `<table>$DeletedTableBackup`. It holds the table as it was at deletion, cannot be deleted, and expires after 35 days. `ListBackups` shows it with `BackupType` `SYSTEM` or `ALL`, and `RestoreTableFromBackup` restores it.
-Deviations from AWS: enabling point-in-time recovery right after `CreateTable` is accepted at once. AWS sometimes refuses it for a few seconds with `ContinuousBackupsUnavailableException`.
+Deviations from AWS: enabling point-in-time recovery right after `CreateTable` is accepted at once. AWS sometimes refuses it for a few seconds with `ContinuousBackupsUnavailableException`. `EarliestRestorableDateTime` is rounded down to the second, as on AWS. A restore or an export at that time on AWS leaves out the writes made earlier in that second, before point-in-time recovery was enabled. Floci keeps changes only from the moment it is enabled, so it includes those writes.
## Kinesis change data capture (CDC)
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 2f2f4de9e2..d0f77cdd7d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -80,6 +80,9 @@ import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.epochSeconds;
+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.instantOf;
+
@ApplicationScoped
public class DynamoDbService {
@@ -4718,7 +4721,8 @@ public class DynamoDbService {
// --- Export Operations ---
- public ExportDescription exportTable(Map<String, Object> request, String region) {
+ /** As on AWS, an ExportTime after LatestRestorableDateTime is accepted up to now. */
+ public ExportDescription exportTable(Map<String, Object> request, Instant exportTime, String region) {
String tableArn = (String) request.get("TableArn");
String s3Bucket = (String) request.get("S3Bucket");
String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;
@@ -4742,10 +4746,24 @@ public class DynamoDbService {
String tableRegion = ref.region() != null ? ref.region() : region;
String storageKey = regionKey(tableRegion, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ TableDefinition table = requireBackupSource(storageKey, tableArn);
+ Instant startTime = clock.instant().truncatedTo(ChronoUnit.MILLIS);
+ // Seconds, as CreationDateTime is shown, so an EarliestRestorableDateTime in the creation second passes.
+ if (exportTime != null && (exportTime.isBefore(table.getCreationDateTime().truncatedTo(ChronoUnit.SECONDS))
+ || startTime.isBefore(exportTime))) {
+ throw new AwsException("InvalidExportTimeException", "Export Time is invalid", 400);
+ }
+ if (!table.isPointInTimeRecoveryEnabled()) {
+ throw new AwsException("PointInTimeRecoveryUnavailableException",
+ "Point in time recovery is not enabled for table '" + table.getTableName() + "'", 400);
+ }
+ if (exportTime != null
+ && exportTime.isBefore(pointInTimeRecovery.window(withPointInTimeStart(storageKey, table)).earliest())) {
+ throw new AwsException("InvalidExportTimeException",
+ "Point in Time Recovery is not available for specified export time", 400);
+ }
+ Instant time = exportTime != null ? exportTime : startTime;
- long now = Instant.now().getEpochSecond();
var exportId = newJobId();
String exportArn = AwsArnUtils.Arn.of("dynamodb", tableRegion, regionResolver.getAccountId(), "table/" + table.getTableName() + "/export/" + exportId).toString();
@@ -4758,8 +4776,8 @@ public class DynamoDbService {
desc.setS3Prefix(s3Prefix);
desc.setExportFormat(exportFormat);
desc.setExportType("FULL_EXPORT");
- desc.setExportTime(now);
- desc.setStartTime(now);
+ desc.setExportTime(epochSeconds(time));
+ desc.setStartTime(epochSeconds(startTime));
desc.setClientToken(clientToken);
desc.setS3SseAlgorithm(s3SseAlgorithm);
desc.setS3BucketOwner(s3BucketOwner);
@@ -4769,9 +4787,8 @@ public class DynamoDbService {
}
ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(scopedItemsKey(storageKey));
- List<JsonNode> snapshot = tableItems != null
- ? List.copyOf(tableItems.values())
- : List.of();
+ List<JsonNode> snapshot = pointInTimeRecovery.itemsAt(regionResolver.getAccountId(), storageKey,
+ tableItems != null ? tableItems : Map.of(), time);
// The worker gets its own copy so the object serialized into this response never
// changes underneath the handler, and the request account so its writes land there.
@@ -4804,7 +4821,7 @@ public class DynamoDbService {
desc.setExportStatus("FAILED");
desc.setFailureCode("S3NoSuchBucket");
desc.setFailureMessage("The specified bucket does not exist: " + s3Bucket);
- desc.setEndTime(Instant.now().getEpochSecond());
+ desc.setEndTime(epochSeconds(clock.instant()));
if (exportStore != null) {
exportStore.put(exportArn, desc);
}
@@ -4830,9 +4847,8 @@ public class DynamoDbService {
manifestSummaryContent.getBytes(StandardCharsets.UTF_8),
"application/json", Map.of());
- long endTime = Instant.now().getEpochSecond();
desc.setExportStatus("COMPLETED");
- desc.setEndTime(endTime);
+ desc.setEndTime(epochSeconds(clock.instant()));
desc.setItemCount(itemCount);
desc.setBilledSizeBytes(billedSize);
desc.setExportManifest(manifestSummaryKey);
@@ -4847,7 +4863,7 @@ public class DynamoDbService {
desc.setExportStatus("FAILED");
desc.setFailureCode("UNKNOWN");
desc.setFailureMessage(e.getMessage());
- desc.setEndTime(Instant.now().getEpochSecond());
+ desc.setEndTime(epochSeconds(clock.instant()));
if (exportStore != null) {
exportStore.put(exportArn, desc);
}
@@ -4885,11 +4901,11 @@ public class DynamoDbService {
ObjectNode root = objectMapper.createObjectNode();
root.put("version", "2020-06-30");
root.put("exportArn", desc.getExportArn());
- root.put("startTime", Instant.ofEpochSecond(desc.getStartTime()).toString());
+ root.put("startTime", instantOf(desc.getStartTime()).toString());
root.put("endTime", Instant.now().toString());
root.put("tableArn", desc.getTableArn());
root.put("tableId", desc.getTableId());
- root.put("exportTime", Instant.ofEpochSecond(desc.getExportTime()).toString());
+ root.put("exportTime", instantOf(desc.getExportTime()).toString());
root.put("s3Bucket", desc.getS3Bucket());
root.putNull("s3Prefix");
if (desc.getS3Prefix() != null) {
@@ -5379,7 +5395,7 @@ public class DynamoDbService {
desc.setExportStatus("FAILED");
desc.setFailureCode("InterruptedByRestart");
desc.setFailureMessage("The emulator restarted before the export finished");
- desc.setEndTime(now);
+ desc.setEndTime(BigDecimal.valueOf(now));
exports.putForAccount(entry.accountId(), entry.key(), desc);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.java
new file mode 100644
index 0000000000..a08744736b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTimestamps.java
@@ -0,0 +1,17 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+
+final class DynamoDbTimestamps {
+
+ private DynamoDbTimestamps() {}
+
+ static BigDecimal epochSeconds(Instant instant) {
+ return BigDecimal.valueOf(instant.toEpochMilli(), 3);
+ }
+
+ static Instant instantOf(BigDecimal epochSeconds) {
+ return Instant.ofEpochMilli(epochSeconds.movePointRight(3).longValue());
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
index 46cbff0310..ac18b2a578 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
@@ -33,7 +33,6 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
-import java.math.BigDecimal;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@@ -48,6 +47,8 @@ import java.util.Set;
import java.util.function.Supplier;
import java.util.regex.Pattern;
+import static io.github.hectorvent.floci.services.dynamodb.DynamoDbTimestamps.epochSeconds;
+
/**
* DynamoDB JSON protocol handler of the native engine, reached through the native backend
* behind the DynamoDB operation seam rather than directly from the controllers.
@@ -60,6 +61,9 @@ public class NativeDynamoDbJsonHandler {
// The order AWS prints the enum constraint in.
private static final List<String> BACKUP_TYPE_FILTERS = List.of("ALL", "AWS_BACKUP", "USER", "SYSTEM");
private static final int MAX_LIST_BACKUPS_LIMIT = 100;
+ private static final int MIN_BACKUP_ARN_LENGTH = 37;
+ private static final int MAX_BACKUP_ARN_LENGTH = 1024;
+ private static final Instant MAX_TIME_RANGE_BOUND = Instant.ofEpochMilli(99_999_999_999_999L);
private static final int MAX_RECOVERY_PERIOD_DAYS = 35;
private final DynamoDbService dynamoDbService;
@@ -1961,6 +1965,15 @@ public class NativeDynamoDbJsonHandler {
violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "
+ "Member must have value less than or equal to " + MAX_LIST_BACKUPS_LIMIT);
}
+ String exclusiveStartBackupArn = textOrNull(request, "ExclusiveStartBackupArn");
+ if (exclusiveStartBackupArn != null && exclusiveStartBackupArn.length() < MIN_BACKUP_ARN_LENGTH) {
+ violations.add("Value '" + exclusiveStartBackupArn + "' at 'exclusiveStartBackupArn' failed to satisfy "
+ + "constraint: Member must have length greater than or equal to " + MIN_BACKUP_ARN_LENGTH);
+ }
+ if (exclusiveStartBackupArn != null && MAX_BACKUP_ARN_LENGTH < exclusiveStartBackupArn.length()) {
+ violations.add("Value '" + exclusiveStartBackupArn + "' at 'exclusiveStartBackupArn' failed to satisfy "
+ + "constraint: Member must have length less than or equal to " + MAX_BACKUP_ARN_LENGTH);
+ }
String backupType = request.hasNonNull("BackupType") ? request.get("BackupType").asText() : "USER";
if (!BACKUP_TYPE_FILTERS.contains(backupType)) {
violations.add("Value '" + backupType + "' at 'backupType' failed to satisfy constraint: "
@@ -1973,9 +1986,11 @@ public class NativeDynamoDbJsonHandler {
throw new AwsException("ValidationException", "One or more parameter values were invalid: "
+ "Time range lower bound must be less than or equal to upper bound", 400);
}
+ checkTimeRangeBound("TimeRangeLowerBound", lowerBound);
+ checkTimeRangeBound("TimeRangeUpperBound", upperBound);
DynamoDbService.ListBackupsResult result = dynamoDbService.listBackups(tableName, limit, lowerBound,
- upperBound, textOrNull(request, "ExclusiveStartBackupArn"), backupType, region);
+ upperBound, exclusiveStartBackupArn, backupType, region);
ObjectNode response = objectMapper.createObjectNode();
ArrayNode summaries = response.putArray("BackupSummaries");
@@ -2211,13 +2226,21 @@ public class NativeDynamoDbJsonHandler {
return node.hasNonNull(member) ? node.get(member).asText() : null;
}
- private static Instant timestampOrNull(JsonNode request, String member) {
- JsonNode node = request.path(member);
- return node.isNumber() ? Instant.ofEpochMilli(node.decimalValue().movePointRight(3).longValue()) : null;
+ private static void checkTimeRangeBound(String member, Instant bound) {
+ if (bound != null && bound.isBefore(Instant.EPOCH)) {
+ throw new AwsException("ValidationException",
+ "Invalid Request: The supplied " + member + " must not be before " + Instant.EPOCH, 400);
+ }
+ if (bound != null && MAX_TIME_RANGE_BOUND.isBefore(bound)) {
+ throw new AwsException("ValidationException",
+ "Invalid Request: The supplied " + member + " must not be after " + MAX_TIME_RANGE_BOUND, 400);
+ }
}
- private static BigDecimal epochSeconds(Instant instant) {
- return BigDecimal.valueOf(instant.toEpochMilli(), 3);
+ private static Instant timestampOrNull(JsonNode request, String member) {
+ JsonNode node = request.path(member);
+ // AWS rounds the seconds as a double, so a value beyond the long range of milliseconds saturates.
+ return node.isNumber() ? Instant.ofEpochMilli(Math.round(node.doubleValue() * 1000)) : null;
}
private ObjectNode backupDetailsNode(TableBackup backup, String status) {
@@ -3384,7 +3407,7 @@ public class NativeDynamoDbJsonHandler {
? e.getValue().asText() : e.getValue()));
io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =
- dynamoDbService.exportTable(params, region);
+ dynamoDbService.exportTable(params, timestampOrNull(request, "ExportTime"), region);
ObjectNode response = objectMapper.createObjectNode();
response.set("ExportDescription", objectMapper.valueToTree(desc));
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java
index 4ae924b18d..ceed3f3df1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java
@@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.math.BigDecimal;
+
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -35,13 +37,13 @@ public class ExportDescription {
private String exportType;
@JsonProperty("ExportTime")
- private Long exportTime;
+ private BigDecimal exportTime;
@JsonProperty("StartTime")
- private Long startTime;
+ private BigDecimal startTime;
@JsonProperty("EndTime")
- private Long endTime;
+ private BigDecimal endTime;
@JsonProperty("ItemCount")
private Long itemCount;
@@ -91,14 +93,14 @@ public class ExportDescription {
public String getExportType() { return exportType; }
public void setExportType(String exportType) { this.exportType = exportType; }
- public Long getExportTime() { return exportTime; }
- public void setExportTime(Long exportTime) { this.exportTime = exportTime; }
+ public BigDecimal getExportTime() { return exportTime; }
+ public void setExportTime(BigDecimal exportTime) { this.exportTime = exportTime; }
- public Long getStartTime() { return startTime; }
- public void setStartTime(Long startTime) { this.startTime = startTime; }
+ public BigDecimal getStartTime() { return startTime; }
+ public void setStartTime(BigDecimal startTime) { this.startTime = startTime; }
- public Long getEndTime() { return endTime; }
- public void setEndTime(Long endTime) { this.endTime = endTime; }
+ public BigDecimal getEndTime() { return endTime; }
+ public void setEndTime(BigDecimal endTime) { this.endTime = endTime; }
public Long getItemCount() { return itemCount; }
public void setItemCount(Long itemCount) { this.itemCount = itemCount; }
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java
index c7921232c3..30bde37e0b 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/RegionScopedListsIntegrationTest.java
@@ -102,6 +102,13 @@ class RegionScopedListsIntegrationTest {
""".formatted(table))
.when().post("/").then().statusCode(200)
.extract().path("TableDescription.TableArn");
+ given().header("Authorization", auth(HOME, "dynamodb"))
+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateContinuousBackups")
+ .contentType(DYNAMODB_JSON)
+ .body("""
+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}
+ """.formatted(table))
+ .when().post("/").then().statusCode(200);
// The bucket is never created: the export job is recorded before it writes anything, and a
// missing bucket only fails the job later. Export jobs have no delete API.
String exportArn = given().header("Authorization", auth(HOME, "dynamodb"))
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
index 8d9300ea8d..417f188b15 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
@@ -269,6 +269,33 @@ class DynamoDbBackupIntegrationTest {
+ "Time range lower bound must be less than or equal to upper bound"));
}
+ @Test
+ void listBackupsChecksTheTimeRangeLimits() {
+ String before = "Invalid Request: The supplied %s must not be before 1970-01-01T00:00:00Z";
+ String after = "Invalid Request: The supplied %s must not be after 5138-11-16T09:46:39.999Z";
+
+ listBackupsBetween("-0.0005", "99999999999.9994").statusCode(200);
+ listBackupsBetween("-0.001", null)
+ .statusCode(400)
+ .body("message", equalTo(before.formatted("TimeRangeLowerBound")));
+ listBackupsBetween(null, "100000000000")
+ .statusCode(400)
+ .body("message", equalTo(after.formatted("TimeRangeUpperBound")));
+ listBackupsBetween("-1e20", "1e20")
+ .statusCode(400)
+ .body("message", equalTo(before.formatted("TimeRangeLowerBound")));
+ listBackupsBetween("0", "1e400")
+ .statusCode(400)
+ .body("message", equalTo(after.formatted("TimeRangeUpperBound")));
+ listBackupsBetween("1e21", "1e20")
+ .statusCode(400)
+ .body("message", equalTo(after.formatted("TimeRangeLowerBound")));
+ listBackupsBetween("100000000000", "-0.001")
+ .statusCode(400)
+ .body("message", equalTo("One or more parameter values were invalid: "
+ + "Time range lower bound must be less than or equal to upper bound"));
+ }
+
@Test
void listBackupsReportsEveryViolation() {
ddb("ListBackups", "{\"Limit\": 0, \"BackupType\": \"BAD\"}")
@@ -284,6 +311,22 @@ class DynamoDbBackupIntegrationTest {
ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + "x".repeat(40) + "\"}")
.statusCode(400)
.body("message", equalTo("Invalid Backup ARN"));
+ String shortArn = "x".repeat(36);
+ ddb("ListBackups", """
+ {"Limit": 0, "ExclusiveStartBackupArn": "%s", "BackupType": "BAD"}
+ """.formatted(shortArn))
+ .statusCode(400)
+ .body("message", equalTo("3 validation errors detected: Value '0' at 'limit' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 1; Value '" + shortArn
+ + "' at 'exclusiveStartBackupArn' failed to satisfy constraint: Member must have length "
+ + "greater than or equal to 37; Value 'BAD' at 'backupType' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [ALL, AWS_BACKUP, USER, SYSTEM]"));
+ String longArn = "x".repeat(1025);
+ ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + longArn + "\"}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '" + longArn + "' at "
+ + "'exclusiveStartBackupArn' failed to satisfy constraint: Member must have length "
+ + "less than or equal to 1024"));
}
@Test
@@ -755,4 +798,10 @@ class DynamoDbBackupIntegrationTest {
{"BackupArn": "%s", "TargetTableName": "%s", %s}
""".formatted(backupArn, target, overrides));
}
+
+ private static ValidatableResponse listBackupsBetween(String lowerBound, String upperBound) {
+ return ddb("ListBackups", """
+ {"TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}
+ """.formatted(lowerBound, upperBound));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
index e8d45128ba..4c95f667c6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.path.json.JsonPath;
+import io.restassured.response.ValidatableResponse;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -9,9 +11,15 @@ import org.junit.jupiter.api.Test;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
import java.util.zip.GZIPInputStream;
+import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;
+import static io.github.hectorvent.floci.testing.DynamoDbRequests.decimalAt;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
import static org.junit.jupiter.api.Assertions.*;
@@ -24,6 +32,7 @@ class DynamoDbExportIntegrationTest {
private static final String BUCKET_NAME = "export-test-bucket";
private static final String TABLE_ARN =
"arn:aws:dynamodb:us-east-1:000000000000:table/" + TABLE_NAME;
+ private static final String TABLE_ARN_PREFIX = "arn:aws:dynamodb:us-east-1:000000000000:table/";
private static boolean setupDone = false;
@@ -59,6 +68,7 @@ class DynamoDbExportIntegrationTest {
}
""".formatted(TABLE_NAME))
.when().post("/").then().statusCode(200);
+ enablePointInTimeRecovery(TABLE_NAME);
// Insert 3 items
putItem("{\"pk\": {\"S\": \"user-1\"}, \"sk\": {\"S\": \"order-001\"}, \"total\": {\"N\": \"99\"}}");
@@ -175,7 +185,7 @@ class DynamoDbExportIntegrationTest {
}
@Test
- void exportTableToPointInTime_tableNotFound_returnsResourceNotFoundException() {
+ void exportTableToPointInTime_tableNotFound_returnsTableNotFoundException() {
given()
.header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
.contentType(DYNAMODB_CONTENT_TYPE)
@@ -188,7 +198,9 @@ class DynamoDbExportIntegrationTest {
.when().post("/")
.then()
.statusCode(400)
- .body("__type", containsString("ResourceNotFoundException"));
+ .body("__type", containsString("TableNotFoundException"))
+ .body("message", equalTo(
+ "Table not found: arn:aws:dynamodb:us-east-1:000000000000:table/NonExistentTable"));
}
@Test
@@ -249,8 +261,145 @@ class DynamoDbExportIntegrationTest {
}
}
+ @Test
+ void exportTimeExportsTheItemsOfThatTime() throws Exception {
+ String table = "ExportTimeItems";
+ createHashTable(table);
+ BigDecimal earliest = decimalAt(enablePointInTimeRecovery(table),
+ "/ContinuousBackupsDescription/PointInTimeRecoveryDescription/EarliestRestorableDateTime");
+ putHashItem(table, "k1", "v1");
+ BigDecimal t1 = pause();
+ putHashItem(table, "k1", "v2");
+ putHashItem(table, "k2", "v1");
+
+ exportAt(table, "export-time-earliest", earliest).statusCode(200);
+ ValidatableResponse atT1 = exportAt(table, "export-time-t1", t1).statusCode(200);
+ assertEquals(0, t1.compareTo(decimalAt(atT1, "/ExportDescription/ExportTime")));
+ long before = System.currentTimeMillis();
+ ValidatableResponse noTime = ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "export-time-now"}
+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)).statusCode(200);
+ long after = System.currentTimeMillis();
+ BigDecimal startTime = decimalAt(noTime, "/ExportDescription/StartTime");
+ assertEquals(0, startTime.compareTo(decimalAt(noTime, "/ExportDescription/ExportTime")));
+ long startMillis = startTime.movePointRight(3).longValueExact();
+ assertTrue(before <= startMillis && startMillis <= after, before + " <= " + startTime + " <= " + after);
+
+ String atT1Arn = atT1.extract().path("ExportDescription.ExportArn");
+ assertEquals("COMPLETED", pollUntilCompleted(atT1Arn, 10_000));
+ assertEquals(List.of("k1=v1"), exportedItems(atT1Arn, "export-time-t1"));
+ String noTimeArn = noTime.extract().path("ExportDescription.ExportArn");
+ assertEquals("COMPLETED", pollUntilCompleted(noTimeArn, 10_000));
+ assertEquals(List.of("k1=v2", "k2=v1"), exportedItems(noTimeArn, "export-time-now"));
+ }
+
+ @Test
+ void exportTableToPointInTimeChecksInTheAwsOrder() throws Exception {
+ String table = "ExportTimeChecks";
+ long beforeCreate = System.currentTimeMillis();
+ createHashTable(table);
+ long afterCreate = System.currentTimeMillis();
+ BigDecimal future = BigDecimal.valueOf(System.currentTimeMillis() + 60_000, 3);
+
+ expectError(exportAt("ExportTimeMissing", "export-checks", future),
+ "TableNotFoundException", "Table not found: " + TABLE_ARN_PREFIX + "ExportTimeMissing");
+ expectError(exportAt(table, "export-checks", future), "InvalidExportTimeException", "Export Time is invalid");
+ expectError(ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s"}
+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)), "PointInTimeRecoveryUnavailableException",
+ "Point in time recovery is not enabled for table '" + table + "'");
+
+ // EarliestRestorableDateTime is in whole seconds. Enabling in a later second puts it after the creation.
+ while (System.currentTimeMillis() / 1000 <= afterCreate / 1000) {
+ Thread.sleep(10);
+ }
+ BigDecimal earliest = decimalAt(enablePointInTimeRecovery(table),
+ "/ContinuousBackupsDescription/PointInTimeRecoveryDescription/EarliestRestorableDateTime");
+ expectError(exportAt(table, "export-checks", BigDecimal.valueOf(beforeCreate - 1000, 3)),
+ "InvalidExportTimeException", "Export Time is invalid");
+ expectError(exportAt(table, "export-checks", earliest.subtract(new BigDecimal("0.001"))),
+ "InvalidExportTimeException", "Point in Time Recovery is not available for specified export time");
+ expectError(exportAt(table, "export-checks", future), "InvalidExportTimeException", "Export Time is invalid");
+ exportAt(table, "export-checks", earliest).statusCode(200);
+ }
+
+ @Test
+ void exportTimeBeyondTheMillisecondRangeIsInvalid() {
+ String table = "ExportTimeRange";
+ createHashTable(table);
+ enablePointInTimeRecovery(table);
+ BigDecimal nowPlusTwoToThe64Millis = BigDecimal.valueOf(System.currentTimeMillis(), 3)
+ .add(new BigDecimal(BigInteger.TWO.pow(64), 3));
+
+ expectError(exportAt(table, "export-range", nowPlusTwoToThe64Millis),
+ "InvalidExportTimeException", "Export Time is invalid");
+ expectError(ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "ExportTime": 1e400}
+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME)),
+ "InvalidExportTimeException", "Export Time is invalid");
+ }
+
// --- Helpers ---
+ /** Returns a time between the writes before and after the call, in epoch seconds with milliseconds. */
+ private static BigDecimal pause() {
+ long now = System.currentTimeMillis();
+ while (System.currentTimeMillis() <= now) {
+ Thread.onSpinWait();
+ }
+ return BigDecimal.valueOf(now, 3);
+ }
+
+ private static void createHashTable(String table) {
+ ddb("CreateTable", """
+ {"TableName": "%s", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "BillingMode": "PAY_PER_REQUEST"}
+ """.formatted(table)).statusCode(200);
+ }
+
+ private static void putHashItem(String table, String pk, String value) {
+ ddb("PutItem", """
+ {"TableName": "%s", "Item": {"pk": {"S": "%s"}, "v": {"S": "%s"}}}
+ """.formatted(table, pk, value)).statusCode(200);
+ }
+
+ private static ValidatableResponse exportAt(String table, String prefix, BigDecimal exportTime) {
+ return ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "%s", "ExportTime": %s}
+ """.formatted(TABLE_ARN_PREFIX + table, BUCKET_NAME, prefix, exportTime.toPlainString()));
+ }
+
+ private List<String> exportedItems(String exportArn, String prefix) throws Exception {
+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
+ String dataKey = given()
+ .when().get("/" + BUCKET_NAME + "/" + prefix + "/AWSDynamoDB/" + exportId + "/manifest-files.json")
+ .then().statusCode(200)
+ .extract().asString().trim();
+ byte[] gzipData = given()
+ .when().get("/" + BUCKET_NAME + "/" + dataKey)
+ .then().statusCode(200)
+ .extract().asByteArray();
+ List<String> items = new ArrayList<>();
+ for (String line : decompressGzip(gzipData).split("\n")) {
+ JsonPath item = JsonPath.from(line);
+ items.add(item.getString("Item.pk.S") + "=" + item.getString("Item.v.S"));
+ }
+ return items.stream().sorted().toList();
+ }
+
+ private static void expectError(ValidatableResponse response, String type, String message) {
+ response.statusCode(400)
+ .body("__type", containsString(type))
+ .body("message", equalTo(message));
+ }
+
+ private static ValidatableResponse enablePointInTimeRecovery(String table) {
+ return ddb("UpdateContinuousBackups", """
+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}
+ """.formatted(table)).statusCode(200);
+ }
+
private void putItem(String itemJson) {
given()
.header("X-Amz-Target", "DynamoDB_20120810.PutItem")
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java
index cb198f0da5..d7b796cff5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java
@@ -109,6 +109,9 @@ class DynamoDbImportIntegrationTest {
@Test
void importTable_gzip_roundTripsAnExport() {
createCompositeTable("RoundTripSource");
+ dynamo("UpdateContinuousBackups", """
+ {"TableName": "RoundTripSource", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}
+ """).then().statusCode(200);
for (int i = 1; i <= 3; i++) {
dynamo("PutItem", "{\"TableName\": \"RoundTripSource\", \"Item\": {\"pk\": {\"S\": \"user-" + i
+ "\"}, \"sk\": {\"S\": \"order-" + i + "\"}, \"total\": {\"N\": \"" + (i * 10) + "\"}}}")
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
index 8368ad6398..37ee8d506f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
@@ -7,6 +7,7 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.math.BigInteger;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
@@ -186,6 +187,22 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {
restoreToTime(table, "PitrIT-window-checks-earliest", earliest).statusCode(200);
}
+ @Test
+ void restoreDateTimeIsRoundedToTheMillisecondAsADouble() {
+ String table = "PitrIT-rounding";
+ createTable(table);
+ updateContinuousBackups(table, "true").statusCode(200);
+ String nowPlusTwoToThe64Millis = BigDecimal.valueOf(System.currentTimeMillis(), 3)
+ .add(new BigDecimal(BigInteger.TWO.pow(64), 3)).toPlainString();
+
+ assertEquals("+292278994-08-17T07:12:55.807Z", refusedRestoreTime(table, nowPlusTwoToThe64Millis));
+ assertEquals("+292278994-08-17T07:12:55.807Z", refusedRestoreTime(table, "1e400"));
+ assertEquals("-292275055-05-16T16:47:04.192Z", refusedRestoreTime(table, "-1e20"));
+ assertEquals("1970-01-01T00:00:00.001Z", refusedRestoreTime(table, "0.0005"));
+ assertEquals("1970-01-01T00:00:00Z", refusedRestoreTime(table, "-0.0005"));
+ assertEquals("2026-10-22T00:42:24.001Z", refusedRestoreTime(table, "1792629744.0004999"));
+ }
+
@Test
void restoreTableToPointInTimeChecksInTheAwsOrder() {
String table = "PitrIT-checks";
@@ -425,6 +442,16 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {
return ddb("RestoreTableToPointInTime", "{" + members + "}");
}
+ private String refusedRestoreTime(String source, String restoreDateTime) {
+ String message = restore("""
+ "SourceTableName": "%s", "TargetTableName": "PitrIT-never", "RestoreDateTime": %s
+ """.formatted(source, restoreDateTime))
+ .statusCode(400)
+ .body("__type", containsString("InvalidRestoreTimeException"))
+ .extract().path("message");
+ return message.substring("RestoreDateTime '".length(), message.indexOf("' must be within"));
+ }
+
private static void expectError(ValidatableResponse response, String type, String message) {
response.statusCode(400)
.body("__type", containsString(type))
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index ff6229bb8d..809a4fb032 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -4493,6 +4493,19 @@ class DynamoDbServiceTest {
assertEquals("ResourceInUseException", e.getErrorCode());
}
+ @Test
+ void exportTable_whileCreating_returnsTableNotFoundException() {
+ createUsersTableInCreating(service);
+ String tableArn = "arn:aws:dynamodb:us-east-1:000000000000:table/Users";
+ Map<String, Object> request = Map.of("TableArn", tableArn, "S3Bucket", "bucket");
+
+ AwsException e = assertThrows(AwsException.class,
+ () -> service.exportTable(request, Instant.EPOCH, "us-east-1"));
+
+ assertEquals("TableNotFoundException", e.getErrorCode());
+ assertEquals("Table not found: " + tableArn, e.getMessage());
+ }
+
@Test
void validateImportRequest_rejectsUnsupportedFormatWithoutUnsupportedOperationWording() {
AwsException e = assertThrows(AwsException.class,