Light, fluffy, and always free - The AWS Local Emulator alternative
feat(dynamodb): restore backups across regions (#6349)
TATSUNO “Taz” Yasuhiro05d47861383f
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 7e31d8cf26..479d6ba8d2 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -488,7 +488,7 @@ aws dynamodb restore-table-from-backup --backup-arn $BACKUP_ARN \--target-table-name UsersRestored --endpoint-url $AWS_ENDPOINT_URL```-`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept.+`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept. To restore a backup into another region, call `RestoreTableFromBackup` in that region with `SSESpecificationOverride`.Deviations from AWS: a restore is done before the response is sent. The response still says `CREATING` with `RestoreInProgress: true`, as on AWS, but the table is `ACTIVE` right away. So `BackupInUseException` and `TableInUseException` for a restore in progress never happen. `BackupSizeBytes` is always 0. Backups made by AWS Backup (`BackupType` `AWS_BACKUP`) are not emulated.diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 43ae8b7d9b..280187a516 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -11,6 +11,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.Digests;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestScopes;@@ -2966,7 +2967,10 @@ public class DynamoDbService {} catch (IllegalArgumentException e) {throw invalidParameter("sourceTableArn is not a valid ARN");}- if (!"dynamodb".equals(arn.service())) {+ if (arn.partition().isEmpty()) {+ throw invalidParameter("Partition must be non-empty: " + sourceTableArn);+ }+ if (!"dynamodb".equals(arn.service()) || !arn.partition().equals(AwsRegions.partitionFor(region))) {throw invalidParameter("Invalid resource Arn: " + sourceTableArn);}if (!arn.resource().startsWith("table/")) {@@ -2981,11 +2985,21 @@ public class DynamoDbService {throw new AwsException("ValidationException","This action is only supported by accounts that match the resource owner’s account.", 400);}- if (!arn.region().equals(region) && !sseOverride) {+ requireSseOverrideAcrossRegions(arn.region(), region, sseOverride);+ return new DynamoDbTableNames.ResolvedTableRef(tableName, arn.region());+ }++ static void requireSseOverrideAcrossRegions(String sourceRegion, String region, boolean sseOverride) {+ if (!sourceRegion.equals(region) && !sseOverride) {throw new AwsException("ValidationException","Invalid Request: sseSpecificationOverride must be provided for cross-region restores", 400);}- return new DynamoDbTableNames.ResolvedTableRef(tableName, arn.region());+ }++ static void requireSourceInPartition(String sourceRegion, String region) {+ if (!AwsRegions.partitionFor(sourceRegion).equals(AwsRegions.partitionFor(region))) {+ throw new AwsException("ValidationException", "Invalid Request: Table ARN is invalid.", 400);+ }}private static AwsException invalidParameter(String message) {@@ -5242,6 +5256,15 @@ public class DynamoDbService {"Backup not found: " + backupArn, 400));}+ /** AWS fails with an internal error for a backup of another partition, after it checks the target. */+ public TableBackup restoreSourceBackup(DynamoDbBackupArns.BackupArn arn, String targetTableName, String region) {+ if (arn.isValid() && !arn.partition().equals(AwsRegions.partitionFor(region))) {+ requireRestoreTarget(targetTableName, region);+ throw new AwsException("InternalServerError", "Internal server error", 500);+ }+ return describeBackup(arn.arn(), arn.region());+ }+public TableBackup deleteBackup(String backupArn, String region) {TableBackup backup = describeBackup(backupArn, region);if ("SYSTEM".equals(backup.getBackupType())) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex f5416037b8..e36d512cf8 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -2015,11 +2015,13 @@ public class NativeDynamoDbJsonHandler {*/private Response handleRestoreTableFromBackup(JsonNode request, String region) {String backupArn = textOrNull(request, "BackupArn");- dynamoDbService.requireOwnBackupArn(backupArn);+ DynamoDbBackupArns.BackupArn arn = dynamoDbService.requireOwnBackupArn(backupArn);String targetTableName = requireTableNameParameter(request, "TargetTableName");throwValidationErrors(restoreViolations(request, targetTableName));+ DynamoDbService.requireSseOverrideAcrossRegions(arn.region(), region,+ request.path("SSESpecificationOverride").isObject());- TableBackup backup = dynamoDbService.describeBackup(backupArn, region);+ TableBackup backup = dynamoDbService.restoreSourceBackup(arn, targetTableName, region);TableDefinition table = tableService.restoreTable(DynamoDbService.snapshotOf(backup.getTable()),targetTableName, parseRestoreOverrides(request), region);ObjectNode description = tableToNode(table);@@ -2063,6 +2065,7 @@ public class NativeDynamoDbJsonHandler {throw new AwsException("ValidationException","Invalid Request: Must provide exactly one of: sourceTableArn, sourceTableName", 400);}+ DynamoDbService.requireSourceInPartition(source.region(), region);boolean useLatestRestorableTime = request.path("UseLatestRestorableTime").asBoolean(false);Instant restoreDateTime = timestampOrNull(request, "RestoreDateTime");if (useLatestRestorableTime && restoreDateTime != null) {diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.javaindex ea3291f57a..d338c5c9f3 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java@@ -25,6 +25,9 @@ class DynamoDbBackupIntegrationTest {private static final String OTHER_REGION_AUTH ="AWS4-HMAC-SHA256 Credential=AKID/20260215/eu-west-2/dynamodb/aws4_request, "+ "SignedHeaders=host, Signature=abc";+ private static final String OTHER_PARTITION_AUTH =+ "AWS4-HMAC-SHA256 Credential=AKID/20260215/cn-north-1/dynamodb/aws4_request, "+ + "SignedHeaders=host, Signature=abc";private static final String OTHER_ACCOUNT_AUTH ="AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/dynamodb/aws4_request, "+ "SignedHeaders=host, Signature=abc";@@ -563,6 +566,69 @@ class DynamoDbBackupIntegrationTest {.body("message", equalTo("Invalid Request: BackupArn is not valid"));}+ @Test+ void restoreAcrossRegionsNeedsAnSseOverride() {+ String table = "BackupIT-cross-region";+ createPlainTable(table);+ putItem(table, "k1");+ String backupArn = createBackup(table, "cross-region");+ String sse = ", \"SSESpecificationOverride\": {\"Enabled\": true, \"SSEType\": \"KMS\"}";+ String missing = TABLE_ARN_PREFIX + table + "/backup/01700000000000-abcdef12";+ String badId = TABLE_ARN_PREFIX + table + "/backup/1700000000000-abcdef12";++ for (String request : List.of(restoreRequest(backupArn, ""), restoreRequest(missing, ""),+ restoreRequest(badId, ""), restoreRequest(backupArn, ", \"SSESpecificationOverride\": null"))) {+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", request)+ .statusCode(400)+ .body("message", equalTo(+ "Invalid Request: sseSpecificationOverride must be provided for cross-region restores"));+ }+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", restoreRequest(badId, sse))+ .statusCode(400)+ .body("message", equalTo("Invalid Request: BackupArn is not valid"));+ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", restoreRequest(missing, sse))+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"));++ signedWith(OTHER_REGION_AUTH, "RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "BackupIT-cross-region-copy"%s}+ """.formatted(backupArn, sse))+ .statusCode(200)+ .body("TableDescription.TableArn",+ equalTo("arn:aws:dynamodb:eu-west-2:000000000000:table/BackupIT-cross-region-copy"))+ .body("TableDescription.RestoreSummary.SourceBackupArn", equalTo(backupArn))+ .body("TableDescription.RestoreSummary.SourceTableArn", equalTo(TABLE_ARN_PREFIX + table))+ .body("TableDescription.SSEDescription.KMSMasterKeyArn", startsWith("arn:aws:kms:eu-west-2:"));+ signedWith(OTHER_REGION_AUTH, "Scan", "{\"TableName\": \"BackupIT-cross-region-copy\"}")+ .statusCode(200)+ .body("Items.pk.S", contains("k1"));+ }++ @Test+ void restoreOfABackupInAnotherPartitionFails() {+ String table = "BackupIT-other-partition";+ signedWith(OTHER_PARTITION_AUTH, "CreateTable", plainTableBody(table)).statusCode(200);+ String backupArn = signedWith(OTHER_PARTITION_AUTH, "CreateBackup",+ "{\"TableName\": \"" + table + "\", \"BackupName\": \"other-partition\"}")+ .statusCode(200)+ .extract().path("BackupDetails.BackupArn");+ String target = "BackupIT-other-partition-target";+ createPlainTable(target);+ String sse = "\"SSESpecificationOverride\": {\"Enabled\": true}";++ ddb("RestoreTableFromBackup", restoreRequest(backupArn, ""))+ .statusCode(400)+ .body("message", equalTo(+ "Invalid Request: sseSpecificationOverride must be provided for cross-region restores"));+ restore(backupArn, target, sse)+ .statusCode(400)+ .body("__type", containsString("TableAlreadyExistsException"));+ ddb("RestoreTableFromBackup", restoreRequest(backupArn, ", " + sse))+ .statusCode(500)+ .body("__type", containsString("InternalServerError"))+ .body("message", equalTo("Internal server error"));+ }+@Testvoid restoreReportsEveryViolationBeforeTheBackupLookup() {String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";@@ -865,14 +931,18 @@ class DynamoDbBackupIntegrationTest {}private void createPlainTable(String table) {- ddb("CreateTable", """+ ddb("CreateTable", plainTableBody(table)).statusCode(200);+ }++ private static String plainTableBody(String table) {+ return """{"TableName": "%s","KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],"AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],"ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}}- """.formatted(table)).statusCode(200);+ """.formatted(table);}private void putItem(String table, String pk) {@@ -899,6 +969,19 @@ class DynamoDbBackupIntegrationTest {""".formatted(backupArn, target, overrides));}+ private static String restoreRequest(String backupArn, String overrides) {+ return "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"BackupIT-never\"" + overrides + "}";+ }++ private static ValidatableResponse signedWith(String authorization, String action, String body) {+ return given().header("Authorization", authorization)+ .header("X-Amz-Target", "DynamoDB_20120810." + action)+ .contentType(CONTENT_TYPE)+ .body(body)+ .when().post("/")+ .then();+ }+private static ValidatableResponse listBackupsBetween(String lowerBound, String upperBound) {return ddb("ListBackups", """{"TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.javaindex 37ee8d506f..66f64ffda6 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryIntegrationTest.java@@ -277,6 +277,12 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {"One or more parameter values were invalid: ARNs must start with 'arn:': abc");expectError(restore("\"SourceTableArn\": \"arn:aws:s3:::bucket\"" + rest), "ValidationException","One or more parameter values were invalid: Invalid resource Arn: arn:aws:s3:::bucket");+ String emptyPartition = "arn::dynamodb:us-east-1:000000000000:table/" + table;+ expectError(restore("\"SourceTableArn\": \"" + emptyPartition + "\"" + rest), "ValidationException",+ "One or more parameter values were invalid: Partition must be non-empty: " + emptyPartition);+ String otherPartition = "arn:aws-cn:dynamodb:cn-north-1:000000000000:table/" + table;+ expectError(restore("\"SourceTableArn\": \"" + otherPartition + "\"" + rest), "ValidationException",+ "One or more parameter values were invalid: Invalid resource Arn: " + otherPartition);expectError(restore("\"SourceTableArn\": \"arn:aws:dynamodb:us-east-1:000000000000:foo/bar\"" + rest),"ValidationException", "One or more parameter values were invalid: sourceTableArn is not a valid ARN");expectError(restore("\"SourceTableArn\": \"" + TABLE_ARN_PREFIX + "ab\"" + rest), "ValidationException",@@ -294,6 +300,10 @@ class DynamoDbPointInTimeRecoveryIntegrationTest {+ "\"TargetTableName\": \"PitrIT-never\", \"UseLatestRestorableTime\": true, "+ "\"SSESpecificationOverride\": {\"Enabled\": true, \"SSEType\": \"KMS\"}"),"TableNotFoundException", "Table not found");+ expectError(restore("\"SourceTableArn\": \"arn:aws:dynamodb:cn-north-1:000000000000:table/PitrIT-missing\", "+ + "\"TargetTableName\": \"PitrIT-never\", \"UseLatestRestorableTime\": true, "+ + "\"SSESpecificationOverride\": {\"Enabled\": true}"),+ "ValidationException", "Invalid Request: Table ARN is invalid.");}@Test