Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): restore backups across regions (#6349)

TATSUNO “Taz” Yasuhiro05d47861383f
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 7e31d8cf26..479d6ba8d2 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -488,7 +488,7 @@ aws dynamodb restore-table-from-backup --backup-arn $BACKUP_ARN \
--target-table-name UsersRestored --endpoint-url $AWS_ENDPOINT_URL
```
-`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept.
+`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept. To restore a backup into another region, call `RestoreTableFromBackup` in that region with `SSESpecificationOverride`.
Deviations from AWS: a restore is done before the response is sent. The response still says `CREATING` with `RestoreInProgress: true`, as on AWS, but the table is `ACTIVE` right away. So `BackupInUseException` and `TableInUseException` for a restore in progress never happen. `BackupSizeBytes` is always 0. Backups made by AWS Backup (`BackupType` `AWS_BACKUP`) are not emulated.
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 43ae8b7d9b..280187a516 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -11,6 +11,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.Digests;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.common.RequestScopes;
@@ -2966,7 +2967,10 @@ public class DynamoDbService {
} catch (IllegalArgumentException e) {
throw invalidParameter("sourceTableArn is not a valid ARN");
}
- if (!"dynamodb".equals(arn.service())) {
+ if (arn.partition().isEmpty()) {
+ throw invalidParameter("Partition must be non-empty: " + sourceTableArn);
+ }
+ if (!"dynamodb".equals(arn.service()) || !arn.partition().equals(AwsRegions.partitionFor(region))) {
throw invalidParameter("Invalid resource Arn: " + sourceTableArn);
}
if (!arn.resource().startsWith("table/")) {
@@ -2981,11 +2985,21 @@ public class DynamoDbService {
throw new AwsException("ValidationException",
"This action is only supported by accounts that match the resource owner’s account.", 400);
}
- if (!arn.region().equals(region) && !sseOverride) {
+ requireSseOverrideAcrossRegions(arn.region(), region, sseOverride);
+ return new DynamoDbTableNames.ResolvedTableRef(tableName, arn.region());
+ }
+
+ static void requireSseOverrideAcrossRegions(String sourceRegion, String region, boolean sseOverride) {
+ if (!sourceRegion.equals(region) && !sseOverride) {
throw new AwsException("ValidationException",
"Invalid Request: sseSpecificationOverride must be provided for cross-region restores", 400);
}
- return new DynamoDbTableNames.ResolvedTableRef(tableName, arn.region());
+ }
+
+ static void requireSourceInPartition(String sourceRegion, String region) {
+ if (!AwsRegions.partitionFor(sourceRegion).equals(AwsRegions.partitionFor(region))) {
+ throw new AwsException("ValidationException", "Invalid Request: Table ARN is invalid.", 400);
+ }
}
private static AwsException invalidParameter(String message) {
@@ -5242,6 +5256,15 @@ public class DynamoDbService {
"Backup not found: " + backupArn, 400));
}
+ /** AWS fails with an internal error for a backup of another partition, after it checks the target. */
+ public TableBackup restoreSourceBackup(DynamoDbBackupArns.BackupArn arn, String targetTableName, String region) {
+ if (arn.isValid() && !arn.partition().equals(AwsRegions.partitionFor(region))) {
+ requireRestoreTarget(targetTableName, region);
+ throw new AwsException("InternalServerError", "Internal server error", 500);
+ }
+ return describeBackup(arn.arn(), arn.region());
+ }
+
public TableBackup deleteBackup(String backupArn, String region) {
TableBackup backup = describeBackup(backupArn, region);
if ("SYSTEM".equals(backup.getBackupType())) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
index f5416037b8..e36d512cf8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
@@ -2015,11 +2015,13 @@ public class NativeDynamoDbJsonHandler {
*/
private Response handleRestoreTableFromBackup(JsonNode request, String region) {
String backupArn = textOrNull(request, "BackupArn");
- dynamoDbService.requireOwnBackupArn(backupArn);
+ DynamoDbBackupArns.BackupArn arn = dynamoDbService.requireOwnBackupArn(backupArn);
String targetTableName = requireTableNameParameter(request, "TargetTableName");
throwValidationErrors(restoreViolations(request, targetTableName));
+ DynamoDbService.requireSseOverrideAcrossRegions(arn.region(), region,
+ request.path("SSESpecificationOverride").isObject());
- TableBackup backup = dynamoDbService.describeBackup(backupArn, region);
+ TableBackup backup = dynamoDbService.restoreSourceBackup(arn, targetTableName, region);
TableDefinition table = tableService.restoreTable(DynamoDbService.snapshotOf(backup.getTable()),
targetTableName, parseRestoreOverrides(request), region);
ObjectNode description = tableToNode(table);
@@ -2063,6 +2065,7 @@ public class NativeDynamoDbJsonHandler {
throw new AwsException("ValidationException",
"Invalid Request: Must provide exactly one of: sourceTableArn, sourceTableName", 400);
}
+ DynamoDbService.requireSourceInPartition(source.region(), region);
boolean useLatestRestorableTime = request.path("UseLatestRestorableTime").asBoolean(false);
Instant restoreDateTime = timestampOrNull(request, "RestoreDateTime");
if (useLatestRestorableTime && restoreDateTime != null) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
index ea3291f57a..d338c5c9f3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
@@ -25,6 +25,9 @@ class DynamoDbBackupIntegrationTest {
private static final String OTHER_REGION_AUTH =
"AWS4-HMAC-SHA256 Credential=AKID/20260215/eu-west-2/dynamodb/aws4_request, "
+ "SignedHeaders=host, Signature=abc";
+ private static final String OTHER_PARTITION_AUTH =
+ "AWS4-HMAC-SHA256 Credential=AKID/20260215/cn-north-1/dynamodb/aws4_request, "
+ + "SignedHeaders=host, Signature=abc";
private static final String OTHER_ACCOUNT_AUTH =
"AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/dynamodb/aws4_request, "
+ "SignedHeaders=host, Signature=abc";
@@ -563,6 +566,69 @@ class DynamoDbBackupIntegrationTest {
.body("message", equalTo("Invalid Request: BackupArn is not valid"));
}
+ @Test
+ void restoreAcrossRegionsNeedsAnSseOverride() {
+ String table = "BackupIT-cross-region";
+ createPlainTable(table);
+ putItem(table, "k1");
+ String backupArn = createBackup(table, "cross-region");
+ String sse = ", \"SSESpecificationOverride\": {\"Enabled\": true, \"SSEType\": \"KMS\"}";
+ String missing = TABLE_ARN_PREFIX + table + "/backup/01700000000000-abcdef12";
+ String badId = TABLE_ARN_PREFIX + table + "/backup/1700000000000-abcdef12";
+
+ for (String request : List.of(restoreRequest(backupArn, ""), restoreRequest(missing, ""),
+ restoreRequest(badId, ""), restoreRequest(backupArn, ", \"SSESpecificationOverride\": null"))) {
+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", request)
+ .statusCode(400)
+ .body("message", equalTo(
+ "Invalid Request: sseSpecificationOverride must be provided for cross-region restores"));
+ }
+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", restoreRequest(badId, sse))
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: BackupArn is not valid"));
+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", restoreRequest(missing, sse))
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"));
+
+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "BackupIT-cross-region-copy"%s}
+ """.formatted(backupArn, sse))
+ .statusCode(200)
+ .body("TableDescription.TableArn",
+ equalTo("arn:aws:dynamodb:eu-west-2:000000000000:table/BackupIT-cross-region-copy"))
+ .body("TableDescription.RestoreSummary.SourceBackupArn", equalTo(backupArn))
+ .body("TableDescription.RestoreSummary.SourceTableArn", equalTo(TABLE_ARN_PREFIX + table))
+ .body("TableDescription.SSEDescription.KMSMasterKeyArn", startsWith("arn:aws:kms:eu-west-2:"));
+ signedWith(OTHER_REGION_AUTH, "Scan", "{\"TableName\": \"BackupIT-cross-region-copy\"}")
+ .statusCode(200)
+ .body("Items.pk.S", contains("k1"));
+ }
+
+ @Test
+ void restoreOfABackupInAnotherPartitionFails() {
+ String table = "BackupIT-other-partition";
+ signedWith(OTHER_PARTITION_AUTH, "CreateTable", plainTableBody(table)).statusCode(200);
+ String backupArn = signedWith(OTHER_PARTITION_AUTH, "CreateBackup",
+ "{\"TableName\": \"" + table + "\", \"BackupName\": \"other-partition\"}")
+ .statusCode(200)
+ .extract().path("BackupDetails.BackupArn");
+ String target = "BackupIT-other-partition-target";
+ createPlainTable(target);
+ String sse = "\"SSESpecificationOverride\": {\"Enabled\": true}";
+
+ ddb("RestoreTableFromBackup", restoreRequest(backupArn, ""))
+ .statusCode(400)
+ .body("message", equalTo(
+ "Invalid Request: sseSpecificationOverride must be provided for cross-region restores"));
+ restore(backupArn, target, sse)
+ .statusCode(400)
+ .body("__type", containsString("TableAlreadyExistsException"));
+ ddb("RestoreTableFromBackup", restoreRequest(backupArn, ", " + sse))
+ .statusCode(500)
+ .body("__type", containsString("InternalServerError"))
+ .body("message", equalTo("Internal server error"));
+ }
+
@Test
void restoreReportsEveryViolationBeforeTheBackupLookup() {
String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";
@@ -865,14 +931,18 @@ class DynamoDbBackupIntegrationTest {
}
private void createPlainTable(String table) {
- ddb("CreateTable", """
+ ddb("CreateTable", plainTableBody(table)).statusCode(200);
+ }
+
+ private static String plainTableBody(String table) {
+ return """
{
"TableName": "%s",
"KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
"AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
"ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}
}
- """.formatted(table)).statusCode(200);
+ """.formatted(table);
}
private void putItem(String table, String pk) {
@@ -899,6 +969,19 @@ class DynamoDbBackupIntegrationTest {
""".formatted(backupArn, target, overrides));
}
+ private static String restoreRequest(String backupArn, String overrides) {
+ return "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"BackupIT-never\"" + overrides + "}";
+ }
+
+ private static ValidatableResponse signedWith(String authorization, String action, String body) {
+ return given().header("Authorization", authorization)
+ .header("X-Amz-Target", "DynamoDB_20120810." + action)
+ .contentType(CONTENT_TYPE)
+ .body(body)
+ .when().post("/")
+ .then();
+ }
+
private static ValidatableResponse listBackupsBetween(String lowerBound, String upperBound) {
return ddb("ListBackups", """
{"TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
index 37ee8d506f..66f64ffda6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java
@@ -277,6 +277,12 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {
"One or more parameter values were invalid: ARNs must start with 'arn:': abc");
expectError(restore("\"SourceTableArn\": \"arn:aws:s3:::bucket\"" + rest), "ValidationException",
"One or more parameter values were invalid: Invalid resource Arn: arn:aws:s3:::bucket");
+ String emptyPartition = "arn::dynamodb:us-east-1:000000000000:table/" + table;
+ expectError(restore("\"SourceTableArn\": \"" + emptyPartition + "\"" + rest), "ValidationException",
+ "One or more parameter values were invalid: Partition must be non-empty: " + emptyPartition);
+ String otherPartition = "arn:aws-cn:dynamodb:cn-north-1:000000000000:table/" + table;
+ expectError(restore("\"SourceTableArn\": \"" + otherPartition + "\"" + rest), "ValidationException",
+ "One or more parameter values were invalid: Invalid resource Arn: " + otherPartition);
expectError(restore("\"SourceTableArn\": \"arn:aws:dynamodb:us-east-1:000000000000:foo/bar\"" + rest),
"ValidationException", "One or more parameter values were invalid: sourceTableArn is not a valid ARN");
expectError(restore("\"SourceTableArn\": \"" + TABLE_ARN_PREFIX + "ab\"" + rest), "ValidationException",
@@ -294,6 +300,10 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {
+ "\"TargetTableName\": \"PitrIT-never\", \"UseLatestRestorableTime\": true, "
+ "\"SSESpecificationOverride\": {\"Enabled\": true, \"SSEType\": \"KMS\"}"),
"TableNotFoundException", "Table not found");
+ expectError(restore("\"SourceTableArn\": \"arn:aws:dynamodb:cn-north-1:000000000000:table/PitrIT-missing\", "
+ + "\"TargetTableName\": \"PitrIT-never\", \"UseLatestRestorableTime\": true, "
+ + "\"SSESpecificationOverride\": {\"Enabled\": true}"),
+ "ValidationException", "Invalid Request: Table ARN is invalid.");
}
@Test