Light, fluffy, and always free - The AWS Local Emulator alternative
feat(dynamodb): add on-demand backups (#6052)
TATSUNO “Taz” Yasuhiro9b59b8997eea
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.javanew file mode 100644index 0000000000..a2dd4e9560--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.java@@ -0,0 +1,120 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;+import software.amazon.awssdk.services.dynamodb.model.*;++import java.util.Map;++import static org.assertj.core.api.Assertions.*;++@DisplayName("DynamoDB On-Demand Backups")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DynamoDbBackupTest {++ private static DynamoDbClient ddb;+ private static final String TABLE_NAME = "sdk-backup-test-table";+ private static final String RESTORED_TABLE_NAME = "sdk-backup-test-restored";+ private static String backupArn;++ @BeforeAll+ static void setup() {+ ddb = TestFixtures.dynamoDbClient();+ ddb.createTable(CreateTableRequest.builder()+ .tableName(TABLE_NAME)+ .attributeDefinitions(AttributeDefinition.builder()+ .attributeName("pk").attributeType(ScalarAttributeType.S).build())+ .keySchema(KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build())+ .billingMode(BillingMode.PAY_PER_REQUEST)+ .build());+ ddb.putItem(PutItemRequest.builder()+ .tableName(TABLE_NAME)+ .item(Map.of("pk", AttributeValue.fromS("backed-up")))+ .build());+ }++ @AfterAll+ static void cleanup() {+ for (String table : new String[] {TABLE_NAME, RESTORED_TABLE_NAME}) {+ try {+ ddb.deleteTable(DeleteTableRequest.builder().tableName(table).build());+ } catch (ResourceNotFoundException ignored) {+ // The restore test did not run.+ }+ }+ if (backupArn != null) {+ try {+ ddb.deleteBackup(DeleteBackupRequest.builder().backupArn(backupArn).build());+ } catch (BackupNotFoundException ignored) {+ // The delete test removed it before it failed.+ }+ }+ if (ddb != null) {+ ddb.close();+ }+ }++ @Test+ @Order(1)+ void createBackup_returnsCreating() {+ BackupDetails details = ddb.createBackup(CreateBackupRequest.builder()+ .tableName(TABLE_NAME).backupName("sdk_backup").build()).backupDetails();++ backupArn = details.backupArn();+ assertThat(backupArn).contains(":table/" + TABLE_NAME + "/backup/");+ assertThat(details.backupStatus()).isEqualTo(BackupStatus.CREATING);+ assertThat(details.backupType()).isEqualTo(BackupType.USER);+ assertThat(details.backupCreationDateTime()).isNotNull();+ }++ @Test+ @Order(2)+ void describeBackup_returnsAvailableWithSourceTable() {+ BackupDescription description = ddb.describeBackup(DescribeBackupRequest.builder()+ .backupArn(backupArn).build()).backupDescription();++ assertThat(description.backupDetails().backupStatus()).isEqualTo(BackupStatus.AVAILABLE);+ assertThat(description.sourceTableDetails().tableName()).isEqualTo(TABLE_NAME);+ assertThat(description.sourceTableDetails().itemCount()).isEqualTo(1L);+ assertThat(description.sourceTableDetails().billingMode()).isEqualTo(BillingMode.PAY_PER_REQUEST);+ }++ @Test+ @Order(3)+ void listBackups_containsTheBackup() {+ ListBackupsResponse response = ddb.listBackups(ListBackupsRequest.builder().tableName(TABLE_NAME).build());++ assertThat(response.backupSummaries()).extracting(BackupSummary::backupArn).containsExactly(backupArn);+ assertThat(response.lastEvaluatedBackupArn()).isNull();+ }++ @Test+ @Order(4)+ void restoreTableFromBackup_restoresTheItems() {+ TableDescription description = ddb.restoreTableFromBackup(RestoreTableFromBackupRequest.builder()+ .backupArn(backupArn).targetTableName(RESTORED_TABLE_NAME).build()).tableDescription();++ assertThat(description.tableName()).isEqualTo(RESTORED_TABLE_NAME);+ assertThat(description.restoreSummary().sourceBackupArn()).isEqualTo(backupArn);+ assertThat(description.restoreSummary().restoreInProgress()).isTrue();++ ddb.waiter().waitUntilTableExists(DescribeTableRequest.builder().tableName(RESTORED_TABLE_NAME).build());+ GetItemResponse item = ddb.getItem(GetItemRequest.builder()+ .tableName(RESTORED_TABLE_NAME)+ .key(Map.of("pk", AttributeValue.fromS("backed-up")))+ .build());+ assertThat(item.item()).containsKey("pk");+ }++ @Test+ @Order(5)+ void deleteBackup_thenDescribeFails() {+ BackupDescription deleted = ddb.deleteBackup(DeleteBackupRequest.builder()+ .backupArn(backupArn).build()).backupDescription();+ assertThat(deleted.backupDetails().backupStatus()).isEqualTo(BackupStatus.DELETED);++ assertThatThrownBy(() -> ddb.describeBackup(DescribeBackupRequest.builder().backupArn(backupArn).build()))+ .isInstanceOf(BackupNotFoundException.class);+ backupArn = null;+ }+}diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex a92e5b97dd..f36e15365d 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -30,6 +30,11 @@| `ListTagsOfResource` | List tags || `DescribeContinuousBackups` | Get PITR backup configuration || `UpdateContinuousBackups` | Enable/disable PITR |+| `CreateBackup` | Take an on-demand backup of a table |+| `DescribeBackup` | Get a backup and the table it was taken from |+| `ListBackups` | List backups by table, type and creation time |+| `DeleteBackup` | Delete a backup |+| `RestoreTableFromBackup` | Create a new table from a backup || `DescribeKinesisStreamingDestination` | List Kinesis streaming destinations || `EnableKinesisStreamingDestination` | Enable Kinesis streaming for a table || `DisableKinesisStreamingDestination` | Disable Kinesis streaming for a table |@@ -447,6 +452,22 @@ The import reads every object under the key prefix. Point it at the `data/` prefDeviations from AWS: only `InputFormat` `DYNAMODB_JSON` with `InputCompressionType` `NONE` or `GZIP` is accepted. `CSV`, `ION` and `ZSTD` are rejected with a `ValidationException`.+## On-demand backups++`CreateBackup` copies the items and settings of a table at the time of the call. The answer says `CREATING`, and the backup is `AVAILABLE` at once. A backup outlives its table. `ListBackups` returns backups in creation order and sets `LastEvaluatedBackupArn` only when more backups remain.++```bash+BACKUP_ARN=$(aws dynamodb create-backup --table-name Users --backup-name nightly \+ --query BackupDetails.BackupArn --output text --endpoint-url $AWS_ENDPOINT_URL)++aws dynamodb restore-table-from-backup --backup-arn $BACKUP_ARN \+ --target-table-name UsersRestored --endpoint-url $AWS_ENDPOINT_URL+```++`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept.++Deviations from AWS: a restore is done before the response is sent. The response still says `CREATING` with `RestoreInProgress: true`, as on AWS, but the table is `ACTIVE` right away. So `BackupInUseException` and `TableInUseException` for a restore in progress never happen. `BackupSizeBytes` is always 0. Backups made by AWS Backup (`BackupType` `AWS_BACKUP`) are not emulated.+## Kinesis change data capture (CDC)diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.javanew file mode 100644index 0000000000..3770ffe070--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.java@@ -0,0 +1,60 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.AwsRegions;++import java.time.Instant;+import java.util.UUID;+import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * Backup ARNs: {@code <table ARN>/backup/<14-digit epoch millis>-<8 hex digits>}.+ */+final class DynamoDbBackupArns {++ private static final Pattern RESOURCE = Pattern.compile("table/([a-zA-Z0-9_.\\-]+)/backup/([^/]{3,})");+ private static final Pattern BACKUP_ID = Pattern.compile("\\d{14}-[0-9a-fA-F]{8}");++ private DynamoDbBackupArns() {}++ record BackupArn(String arn, String region, String accountId, String tableName, long creationMillis) {}++ /**+ * AWS rejects an ARN it cannot split into a table and a backup id with "Invalid Backup ARN", and+ * one that splits but names another service, an unknown region or a malformed id with+ * "Invalid Request: BackupArn is not valid".+ */+ static BackupArn parse(String input) {+ if (input == null) {+ throw invalidBackupArn();+ }+ AwsArnUtils.Arn arn;+ try {+ arn = AwsArnUtils.parse(input);+ } catch (IllegalArgumentException e) {+ throw invalidBackupArn();+ }+ Matcher resource = RESOURCE.matcher(arn.resource());+ if (!resource.matches()) {+ throw invalidBackupArn();+ }+ String backupId = resource.group(2);+ if (!"dynamodb".equals(arn.service()) || !AwsRegions.isRegionId(arn.region())+ || !BACKUP_ID.matcher(backupId).matches()) {+ throw new AwsException("ValidationException", "Invalid Request: BackupArn is not valid", 400);+ }+ long creationMillis = Long.parseLong(backupId.substring(0, 14));+ return new BackupArn(input, arn.region(), arn.accountId(), resource.group(1), creationMillis);+ }++ static String mint(String tableArn, Instant creationDateTime) {+ return tableArn + "/backup/" + String.format("%014d", creationDateTime.toEpochMilli())+ + "-" + UUID.randomUUID().toString().substring(0, 8);+ }++ private static AwsException invalidBackupArn() {+ return new AwsException("ValidationException", "Invalid Backup ARN", 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 78c56ff7b2..cc94d92d32 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.JsonNodeFactory;import com.fasterxml.jackson.databind.node.ObjectNode;+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;@@ -16,6 +17,7 @@ import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.SmithyValidationErrors;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;@@ -28,6 +30,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescriptionimport io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;import io.github.hectorvent.floci.services.s3.S3Service;@@ -46,6 +49,7 @@ import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.time.Duration;import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Collections;import java.util.Comparator;@@ -98,6 +102,7 @@ public class DynamoDbService {private final StorageBackend<String, Map<String, JsonNode>> itemStore;private final StorageBackend<String, ExportDescription> exportStore;private final StorageBackend<String, ImportTableDescription> importStore;+ private final StorageBackend<String, TableBackup> backupStore;// Items stored per table: storageKey -> Map<itemKey, item>// itemKey is "pk" or "pk#sk" depending on table schemaprivate final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();@@ -163,6 +168,8 @@ public class DynamoDbService {new TypeReference<Map<String, ExportDescription>>() {}),storageFactory.create("dynamodb", "dynamodb-imports.json",new TypeReference<Map<String, ImportTableDescription>>() {}),+ storageFactory.create("dynamodb", "dynamodb-backups.json",+ new TypeReference<Map<String, TableBackup>>() {}),regionResolver, streamService, kinesisForwarder, s3Service, objectMapper,config.services().dynamodb().vectorIndexAllocationSeconds(),config.services().dynamodb().vectorIndexBackfillSeconds());@@ -215,10 +222,28 @@ public class DynamoDbService {ObjectMapper objectMapper,int vectorIndexAllocationSeconds,int vectorIndexBackfillSeconds) {+ this(tableStore, itemStore, exportStore, importStore, new InMemoryStorage<>(), regionResolver,+ streamService, kinesisForwarder, s3Service, objectMapper,+ vectorIndexAllocationSeconds, vectorIndexBackfillSeconds);+ }++ DynamoDbService(StorageBackend<String, TableDefinition> tableStore,+ StorageBackend<String, Map<String, JsonNode>> itemStore,+ StorageBackend<String, ExportDescription> exportStore,+ StorageBackend<String, ImportTableDescription> importStore,+ StorageBackend<String, TableBackup> backupStore,+ RegionResolver regionResolver,+ DynamoDbStreamService streamService,+ KinesisStreamingForwarder kinesisForwarder,+ S3Service s3Service,+ ObjectMapper objectMapper,+ int vectorIndexAllocationSeconds,+ int vectorIndexBackfillSeconds) {this.tableStore = tableStore;this.itemStore = itemStore;this.exportStore = exportStore;this.importStore = importStore;+ this.backupStore = backupStore;this.regionResolver = regionResolver;this.streamService = streamService;this.kinesisForwarder = kinesisForwarder;@@ -4781,6 +4806,124 @@ public class DynamoDbService {return System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");}+ // --- Backup Operations ---++ // The live table object is shared and later mutated in place, so a backup keeps its own copy.+ private static final ObjectMapper SNAPSHOT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());++ private static final Comparator<TableBackup> BACKUP_ORDER = Comparator+ .comparing(TableBackup::getCreationDateTime)+ .thenComparing(TableBackup::getBackupArn);++ public record ListBackupsResult(List<TableBackup> backups, String lastEvaluatedBackupArn) {}++ public TableBackup createBackup(String tableName, String backupName, String region) {+ String canonicalTableName = canonicalTableName(region, tableName);+ String storageKey = regionKey(region, canonicalTableName);+ TableDefinition table = tableStore.get(storageKey)+ .filter(t -> !"CREATING".equals(t.getTableStatus()))+ .orElseThrow(() -> new AwsException("TableNotFoundException",+ "Table not found: " + canonicalTableName, 400));+ ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scopedItemsKey(storageKey));+ List<JsonNode> snapshot = new ArrayList<>();+ if (items != null) {+ items.values().forEach(item -> snapshot.add(item.deepCopy()));+ }+ TableDefinition source = snapshotOf(table);+ source.setItemCount(snapshot.size());++ Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS);+ TableBackup backup = new TableBackup();+ backup.setBackupArn(DynamoDbBackupArns.mint(table.getTableArn(), now));+ backup.setBackupName(backupName);+ backup.setBackupType("USER");+ backup.setCreationDateTime(now);+ backup.setSizeBytes(source.getTableSizeBytes());+ backup.setRegion(region);+ backup.setTable(source);+ backup.setItems(snapshot);+ backupStore.put(backup.getBackupArn(), backup);+ LOG.infov("Created backup {0} of table {1}", backup.getBackupArn(), canonicalTableName);+ return backup;+ }++ public TableBackup describeBackup(String backupArn, String region) {+ DynamoDbBackupArns.BackupArn arn = DynamoDbBackupArns.parse(backupArn);+ if (!arn.accountId().equals(regionResolver.getAccountId())) {+ throw new AwsException("AccessDeniedException", "Access is denied", 400);+ }+ return backupStore.get(backupArn)+ .filter(backup -> backup.getRegion().equals(region))+ .orElseThrow(() -> new AwsException("BackupNotFoundException",+ "Backup not found: " + backupArn, 400));+ }++ public TableBackup deleteBackup(String backupArn, String region) {+ TableBackup backup = describeBackup(backupArn, region);+ backupStore.delete(backupArn);+ LOG.infov("Deleted backup {0}", backupArn);+ return backup;+ }++ /**+ * Backups in creation order. AWS resumes after the position an ExclusiveStartBackupArn names,+ * even when no such backup exists, so the position comes from the ARN itself.+ */+ public ListBackupsResult listBackups(String tableName, Integer limit, Instant lowerBound, Instant upperBound,+ String exclusiveStartBackupArn, String backupType, String region) {+ DynamoDbBackupArns.BackupArn start = exclusiveStartBackupArn != null+ ? DynamoDbBackupArns.parse(exclusiveStartBackupArn) : null;+ Set<String> types = "ALL".equals(backupType) ? Set.of("USER", "SYSTEM") : Set.of(backupType);+ List<TableBackup> matching = backupStore.scan(key -> true).stream()+ .filter(backup -> backup.getRegion().equals(region))+ .filter(backup -> tableName == null || tableName.equals(backup.getTable().getTableName()))+ .filter(backup -> types.contains(backup.getBackupType()))+ .filter(backup -> lowerBound == null || !backup.getCreationDateTime().isBefore(lowerBound))+ .filter(backup -> upperBound == null || backup.getCreationDateTime().isBefore(upperBound))+ .filter(backup -> start == null || isAfter(backup, start))+ .sorted(BACKUP_ORDER)+ .toList();+ if (limit == null || matching.size() <= limit) {+ return new ListBackupsResult(matching, null);+ }+ List<TableBackup> page = matching.subList(0, limit);+ return new ListBackupsResult(page, page.getLast().getBackupArn());+ }++ static TableDefinition snapshotOf(TableDefinition table) {+ return SNAPSHOT_MAPPER.convertValue(table, TableDefinition.class);+ }++ void requireRestoreTarget(String tableName, String region) {+ tableStore.get(regionKey(region, tableName)).ifPresent(table -> {+ if ("CREATING".equals(table.getTableStatus())) {+ throw new AwsException("TableInUseException",+ "Table is in the process of being created: " + tableName, 400);+ }+ throw new AwsException("TableAlreadyExistsException", "Table already exists: " + tableName, 400);+ });+ }++ /** Loads the restored items into the table a restore created and makes it ACTIVE. */+ public void finishRestore(String tableName, List<JsonNode> items, String region) {+ String storageKey = regionKey(region, tableName);+ TableDefinition table = tableStore.get(storageKey)+ .orElseThrow(() -> resourceNotFoundException(tableName));+ ConcurrentSkipListMap<String, JsonNode> restored = new ConcurrentSkipListMap<>();+ for (JsonNode item : items) {+ restored.put(buildItemKey(table, item), item.deepCopy());+ }+ itemsByTable.put(scopedItemsKey(storageKey), restored);+ persistItems(storageKey);+ activateTable(tableName, region);+ LOG.infov("Restored {0} items into table {1}", items.size(), tableName);+ }++ private static boolean isAfter(TableBackup backup, DynamoDbBackupArns.BackupArn start) {+ int byTime = Long.compare(backup.getCreationDateTime().toEpochMilli(), start.creationMillis());+ return 0 < byTime || (byTime == 0 && 0 < backup.getBackupArn().compareTo(start.arn()));+ }+// --- Import Operations ---/**diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex 035db086c8..791dc03bbd 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -10,8 +10,10 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.SmithyValidationErrors;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.CreateTableRequest;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.GsiThroughputUpdate;+import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.IndexOverride;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.KinesisDestination;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.OnDemandThroughput;+import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.RestoreOverrides;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.TableSettings;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.Throughput;import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.UpdateTableRequest;@@ -23,6 +25,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.SearchSchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;import io.github.hectorvent.floci.services.kinesis.KinesisService;@@ -30,6 +33,8 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import jakarta.ws.rs.core.Response;+import java.math.BigDecimal;+import java.time.Instant;import java.util.ArrayList;import java.util.Collections;import java.util.HashMap;@@ -52,6 +57,9 @@ public class NativeDynamoDbJsonHandler {private static final String SSE_TYPE_KMS = "KMS";private static final Pattern INDEX_NAME_CHARS = Pattern.compile("[a-zA-Z0-9_.\\-]+");+ // The order AWS prints the enum constraint in.+ private static final List<String> BACKUP_TYPE_FILTERS = List.of("ALL", "AWS_BACKUP", "USER", "SYSTEM");+ private static final int MAX_LIST_BACKUPS_LIMIT = 100;private final DynamoDbService dynamoDbService;private final NativeDynamoDbTableService tableService;@@ -95,6 +103,11 @@ public class NativeDynamoDbJsonHandler {case "UpdateTimeToLive" -> handleUpdateTimeToLive(request, region);case "DescribeContinuousBackups" -> handleDescribeContinuousBackups(request, region);case "UpdateContinuousBackups" -> handleUpdateContinuousBackups(request, region);+ case "CreateBackup" -> handleCreateBackup(request, region);+ case "DescribeBackup" -> handleDescribeBackup(request, region);+ case "ListBackups" -> handleListBackups(request, region);+ case "DeleteBackup" -> handleDeleteBackup(request, region);+ case "RestoreTableFromBackup" -> handleRestoreTableFromBackup(request, region);case "TransactWriteItems" -> handleTransactWriteItems(request, region);case "TransactGetItems" -> handleTransactGetItems(request, region);case "TagResource" -> handleTagResource(request, region);@@ -1879,6 +1892,352 @@ public class NativeDynamoDbJsonHandler {return Response.ok(response).build();}+ private Response handleCreateBackup(JsonNode request, String region) {+ String tableName = requireTableNameParameter(request, "TableName");+ DynamoDbTableNames.resolveWithRegion(tableName, region);+ JsonNode backupNameNode = request.path("BackupName");+ if (backupNameNode.isMissingNode() || backupNameNode.isNull()) {+ throwValidationErrors(List.of(+ "Value null at 'backupName' failed to satisfy constraint: Member must not be null"));+ }+ String backupName = backupNameNode.asText();+ throwValidationErrors(nameViolations(backupName, "backupName"));++ TableBackup backup = dynamoDbService.createBackup(tableName, backupName, region);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("BackupDetails", backupDetailsNode(backup, "CREATING"));+ return Response.ok(response).build();+ }++ private Response handleDescribeBackup(JsonNode request, String region) {+ TableBackup backup = dynamoDbService.describeBackup(textOrNull(request, "BackupArn"), region);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("BackupDescription", backupDescriptionNode(backup, "AVAILABLE"));+ return Response.ok(response).build();+ }++ private Response handleDeleteBackup(JsonNode request, String region) {+ TableBackup backup = dynamoDbService.deleteBackup(textOrNull(request, "BackupArn"), region);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("BackupDescription", backupDescriptionNode(backup, "DELETED"));+ return Response.ok(response).build();+ }++ private Response handleListBackups(JsonNode request, String region) {+ String tableName = request.hasNonNull("TableName")+ ? DynamoDbTableNames.resolveWithRegion(request.get("TableName").asText(), region).name()+ : null;+ List<String> violations = new ArrayList<>();+ Integer limit = request.hasNonNull("Limit") ? request.get("Limit").asInt() : null;+ if (limit != null && limit < 1) {+ violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "+ + "Member must have value greater than or equal to 1");+ }+ if (limit != null && MAX_LIST_BACKUPS_LIMIT < limit) {+ violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "+ + "Member must have value less than or equal to " + MAX_LIST_BACKUPS_LIMIT);+ }+ String backupType = request.hasNonNull("BackupType") ? request.get("BackupType").asText() : "USER";+ if (!BACKUP_TYPE_FILTERS.contains(backupType)) {+ violations.add("Value '" + backupType + "' at 'backupType' failed to satisfy constraint: "+ + "Member must satisfy enum value set: " + BACKUP_TYPE_FILTERS);+ }+ throwValidationErrors(violations);+ Instant lowerBound = timestampOrNull(request, "TimeRangeLowerBound");+ Instant upperBound = timestampOrNull(request, "TimeRangeUpperBound");+ if (lowerBound != null && upperBound != null && upperBound.isBefore(lowerBound)) {+ throw new AwsException("ValidationException", "One or more parameter values were invalid: "+ + "Time range lower bound must be less than or equal to upper bound", 400);+ }++ DynamoDbService.ListBackupsResult result = dynamoDbService.listBackups(tableName, limit, lowerBound,+ upperBound, textOrNull(request, "ExclusiveStartBackupArn"), backupType, region);++ ObjectNode response = objectMapper.createObjectNode();+ ArrayNode summaries = response.putArray("BackupSummaries");+ for (TableBackup backup : result.backups()) {+ summaries.add(backupSummaryNode(backup));+ }+ if (result.lastEvaluatedBackupArn() != null) {+ response.put("LastEvaluatedBackupArn", result.lastEvaluatedBackupArn());+ }+ return Response.ok(response).build();+ }++ /**+ * Restores synchronously. The answer still reports the table CREATING with the restore in+ * progress, as AWS does, but the table is ACTIVE with every item by the time it is sent.+ */+ private Response handleRestoreTableFromBackup(JsonNode request, String region) {+ String backupArn = textOrNull(request, "BackupArn");+ DynamoDbBackupArns.parse(backupArn);+ String targetTableName = requireTableNameParameter(request, "TargetTableName");+ List<String> violations = new ArrayList<>(throughputViolations(request.path("ProvisionedThroughputOverride"),+ "provisionedThroughputOverride"));+ JsonNode billingModeOverride = request.path("BillingModeOverride");+ if (billingModeOverride.isTextual() && !"PROVISIONED".equals(billingModeOverride.asText())+ && !"PAY_PER_REQUEST".equals(billingModeOverride.asText())) {+ violations.add("Value '" + billingModeOverride.asText() + "' at 'billingModeOverride' failed to "+ + "satisfy constraint: Member must satisfy enum value set: [PROVISIONED, PAY_PER_REQUEST]");+ }+ violations.addAll(nameViolations(targetTableName, "targetTableName"));+ JsonNode gsiOverrides = request.path("GlobalSecondaryIndexOverride");+ if (gsiOverrides.isArray()) {+ int position = 0;+ for (JsonNode gsiOverride : gsiOverrides) {+ position++;+ String member = "globalSecondaryIndexOverride." + position + ".member";+ violations.addAll(restoreIndexNameViolations(gsiOverride.path("IndexName"), member + ".indexName"));+ violations.addAll(throughputViolations(gsiOverride.path("ProvisionedThroughput"),+ member + ".provisionedThroughput"));+ }+ }+ throwValidationErrors(violations);++ TableBackup backup = dynamoDbService.describeBackup(backupArn, region);+ TableDefinition table = tableService.restoreTable(DynamoDbService.snapshotOf(backup.getTable()),+ targetTableName, parseRestoreOverrides(request), region);+ ObjectNode description = tableToNode(table);+ description.set("RestoreSummary", restoreSummaryNode(backup.getBackupArn(),+ backup.getTable().getTableArn(), backup.getCreationDateTime()));+ dynamoDbService.finishRestore(targetTableName, backup.getItems(), region);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("TableDescription", description);+ return Response.ok(response).build();+ }++ private static RestoreOverrides parseRestoreOverrides(JsonNode request) {+ JsonNode throughputNode = request.path("ProvisionedThroughputOverride");+ Throughput throughput = throughputNode.isObject()+ ? new Throughput(throughputNode.path("ReadCapacityUnits").asLong(),+ throughputNode.path("WriteCapacityUnits").asLong())+ : null;+ List<VectorIndex> vectorIndexes = null;+ JsonNode vectorArray = request.path("VectorIndexOverride");+ if (vectorArray.isArray()) {+ vectorIndexes = new ArrayList<>();+ int position = 0;+ for (JsonNode vectorNode : vectorArray) {+ position++;+ vectorIndexes.add(parseVectorIndex(vectorNode, "vectorIndexOverride." + position + ".member"));+ }+ }+ JsonNode sse = request.path("SSESpecificationOverride");+ return new RestoreOverrides(textOrNull(request, "BillingModeOverride"), throughput,+ parseOnDemandThroughput(request.path("OnDemandThroughputOverride")),+ parseIndexOverrides(request.path("GlobalSecondaryIndexOverride"), "globalSecondaryIndexOverride"),+ parseIndexOverrides(request.path("LocalSecondaryIndexOverride"), "localSecondaryIndexOverride"),+ vectorIndexes,+ sse.isObject() ? sse.path("Enabled").asBoolean(false) : null,+ textOrNull(sse, "SSEType"), textOrNull(sse, "KMSMasterKeyId"));+ }++ private static List<IndexOverride> parseIndexOverrides(JsonNode array, String memberPrefix) {+ if (!array.isArray()) {+ return null;+ }+ List<IndexOverride> overrides = new ArrayList<>();+ int position = 0;+ for (JsonNode node : array) {+ position++;+ String indexName = requireIndexName(node, memberPrefix + "." + position + ".member");+ List<KeySchemaElement> keySchema = new ArrayList<>();+ node.path("KeySchema").forEach(ks -> keySchema.add(+ new KeySchemaElement(ks.path("AttributeName").asText(), ks.path("KeyType").asText())));+ List<String> nonKeyAttributes = new ArrayList<>();+ node.path("Projection").path("NonKeyAttributes").forEach(a -> nonKeyAttributes.add(a.asText()));+ JsonNode throughputNode = node.path("ProvisionedThroughput");+ Throughput throughput = throughputNode.isObject()+ ? new Throughput(throughputNode.path("ReadCapacityUnits").asLong(),+ throughputNode.path("WriteCapacityUnits").asLong())+ : null;+ overrides.add(new IndexOverride(indexName, keySchema,+ node.path("Projection").path("ProjectionType").asText("ALL"), nonKeyAttributes, throughput,+ parseOnDemandThroughput(node.path("OnDemandThroughput"))));+ }+ return overrides;+ }++ /** AWS names the member 'TableName' in this message even when the request calls it something else. */+ private static String requireTableNameParameter(JsonNode request, String member) {+ JsonNode node = request.path(member);+ if (node.isMissingNode() || node.isNull()) {+ throw new AwsException("ValidationException",+ "The parameter 'TableName' is required but was not present in the request", 400);+ }+ return node.asText();+ }++ /** Table, index and backup names share one pattern and length range. */+ private static List<String> nameViolations(String name, String memberPath) {+ String value = "Value '" + name + "' at '" + memberPath + "' failed to satisfy constraint: ";+ List<String> violations = new ArrayList<>();+ if (!INDEX_NAME_CHARS.matcher(name).matches()) {+ violations.add(value + "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+");+ }+ if (name.length() < 3) {+ violations.add(value + "Member must have length greater than or equal to 3");+ }+ if (255 < name.length()) {+ violations.add(value + "Member must have length less than or equal to 255");+ }+ return violations;+ }++ /** AWS lists WriteCapacityUnits before ReadCapacityUnits. */+ private static List<String> throughputViolations(JsonNode throughput, String memberPath) {+ List<String> violations = new ArrayList<>();+ if (!throughput.isObject()) {+ return violations;+ }+ for (String member : List.of("WriteCapacityUnits", "ReadCapacityUnits")) {+ JsonNode value = throughput.path(member);+ String constraint = " at '" + memberPath + "." + Character.toLowerCase(member.charAt(0))+ + member.substring(1) + "' failed to satisfy constraint: ";+ if (value.isMissingNode() || value.isNull()) {+ violations.add("Value null" + constraint + "Member must not be null");+ } else if (value.asLong() < 1) {+ violations.add("Value '" + value.asText() + "'" + constraint+ + "Member must have value greater than or equal to 1");+ }+ }+ return violations;+ }++ /** Unlike CreateTable, a restore lists the length violation of an index name before the pattern. */+ private static List<String> restoreIndexNameViolations(JsonNode indexName, String memberPath) {+ String constraint = " at '" + memberPath + "' failed to satisfy constraint: ";+ if (indexName.isMissingNode() || indexName.isNull()) {+ return List.of("Value null" + constraint + "Member must not be null");+ }+ String value = "Value '" + indexName.asText() + "'" + constraint;+ List<String> violations = new ArrayList<>();+ if (indexName.asText().length() < 3) {+ violations.add(value + "Member must have length greater than or equal to 3");+ }+ if (255 < indexName.asText().length()) {+ violations.add(value + "Member must have length less than or equal to 255");+ }+ if (!INDEX_NAME_CHARS.matcher(indexName.asText()).matches()) {+ violations.add(value + "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+");+ }+ return violations;+ }++ private static String textOrNull(JsonNode node, String member) {+ return node.hasNonNull(member) ? node.get(member).asText() : null;+ }++ private static Instant timestampOrNull(JsonNode request, String member) {+ JsonNode node = request.path(member);+ return node.isNumber() ? Instant.ofEpochMilli(node.decimalValue().movePointRight(3).longValue()) : null;+ }++ private static BigDecimal epochSeconds(Instant instant) {+ return BigDecimal.valueOf(instant.toEpochMilli(), 3);+ }++ private ObjectNode backupDetailsNode(TableBackup backup, String status) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("BackupArn", backup.getBackupArn());+ node.put("BackupName", backup.getBackupName());+ node.put("BackupSizeBytes", backup.getSizeBytes());+ node.put("BackupStatus", status);+ node.put("BackupType", backup.getBackupType());+ node.put("BackupCreationDateTime", epochSeconds(backup.getCreationDateTime()));+ if (backup.getExpiryDateTime() != null) {+ node.put("BackupExpiryDateTime", epochSeconds(backup.getExpiryDateTime()));+ }+ return node;+ }++ private ObjectNode backupSummaryNode(TableBackup backup) {+ ObjectNode node = backupDetailsNode(backup, "AVAILABLE");+ TableDefinition table = backup.getTable();+ node.put("TableName", table.getTableName());+ node.put("TableId", table.getTableId());+ node.put("TableArn", table.getTableArn());+ return node;+ }++ /** The source table parts are DescribeTable's, minus the members that only describe a live table. */+ private ObjectNode backupDescriptionNode(TableBackup backup, String status) {+ TableDefinition table = backup.getTable();+ ObjectNode described = tableToNode(table);++ ObjectNode source = objectMapper.createObjectNode();+ source.put("TableName", table.getTableName());+ source.put("TableId", table.getTableId());+ source.put("TableArn", table.getTableArn());+ source.put("TableSizeBytes", table.getTableSizeBytes());+ source.set("KeySchema", described.get("KeySchema"));+ source.set("TableCreationDateTime", described.get("CreationDateTime"));+ ObjectNode throughput = (ObjectNode) described.get("ProvisionedThroughput");+ throughput.remove("NumberOfDecreasesToday");+ source.set("ProvisionedThroughput", throughput);+ if (described.has("OnDemandThroughput")) {+ source.set("OnDemandThroughput", described.get("OnDemandThroughput"));+ }+ source.put("ItemCount", table.getItemCount());+ source.put("BillingMode", "PAY_PER_REQUEST".equals(table.getBillingMode()) ? "PAY_PER_REQUEST" : "PROVISIONED");++ ObjectNode features = objectMapper.createObjectNode();+ if (described.has("LocalSecondaryIndexes")) {+ for (JsonNode lsi : described.get("LocalSecondaryIndexes")) {+ ((ObjectNode) lsi).remove(List.of("IndexArn", "IndexSizeBytes", "ItemCount"));+ }+ features.set("LocalSecondaryIndexes", described.get("LocalSecondaryIndexes"));+ }+ if (described.has("GlobalSecondaryIndexes")) {+ for (JsonNode gsi : described.get("GlobalSecondaryIndexes")) {+ ((ObjectNode) gsi).remove(List.of("IndexArn", "IndexStatus", "IndexSizeBytes", "ItemCount"));+ ((ObjectNode) gsi.get("ProvisionedThroughput")).remove("NumberOfDecreasesToday");+ }+ features.set("GlobalSecondaryIndexes", described.get("GlobalSecondaryIndexes"));+ }+ if (table.isStreamEnabled()) {+ ObjectNode stream = features.putObject("StreamDescription");+ stream.put("StreamEnabled", true);+ stream.put("StreamViewType", table.getStreamViewType());+ }+ if (table.isTtlEnabled()) {+ ObjectNode ttl = features.putObject("TimeToLiveDescription");+ ttl.put("AttributeName", table.getTtlAttributeName());+ ttl.put("TimeToLiveStatus", "ENABLED");+ }+ if (described.has("SSEDescription")) {+ features.set("SSEDescription", described.get("SSEDescription"));+ }+ if (described.has("VectorIndexes")) {+ for (JsonNode vectorIndex : described.get("VectorIndexes")) {+ ((ObjectNode) vectorIndex).remove(+ List.of("IndexArn", "IndexStatus", "Backfilling", "IndexSizeBytes", "ItemCount"));+ }+ features.set("VectorIndexes", described.get("VectorIndexes"));+ }++ ObjectNode node = objectMapper.createObjectNode();+ node.set("BackupDetails", backupDetailsNode(backup, status));+ node.set("SourceTableDetails", source);+ node.set("SourceTableFeatureDetails", features);+ return node;+ }++ private ObjectNode restoreSummaryNode(String sourceBackupArn, String sourceTableArn, Instant restoreDateTime) {+ ObjectNode node = objectMapper.createObjectNode();+ if (sourceBackupArn != null) {+ node.put("SourceBackupArn", sourceBackupArn);+ }+ node.put("SourceTableArn", sourceTableArn);+ node.put("RestoreDateTime", epochSeconds(restoreDateTime));+ node.put("RestoreInProgress", true);+ return node;+ }+private Response handleTransactWriteItems(JsonNode request, String region) {JsonNode transactItemsNode = requireValidTransactRequest(request);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.javaindex fb287ece19..a9ab0402ab 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java@@ -10,18 +10,21 @@ import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestinimport io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;import io.github.hectorvent.floci.services.kinesis.KinesisService;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.util.ArrayList;+import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;/**- * Table control operations on the native DynamoDB engine: CreateTable, UpdateTable and the Kinesis- * streaming destination toggles.+ * Table control operations on the native DynamoDB engine: CreateTable, UpdateTable, the table a+ * restore creates and the Kinesis streaming destination toggles.** <p>Owns the sequence each operation runs against the backend: the validation that must fail* before any side effect, the underlying create or update, the settings applied to the stored@@ -74,6 +77,18 @@ public class NativeDynamoDbTableService {record KinesisDestination(String tableName, String streamArn, String destinationStatus,String approximateCreationDateTimePrecision) {}+ /** One GlobalSecondaryIndexOverride or LocalSecondaryIndexOverride entry; LSIs carry no throughput. */+ record IndexOverride(String indexName, List<KeySchemaElement> keySchema, String projectionType,+ List<String> nonKeyAttributes, Throughput provisionedThroughput,+ OnDemandThroughput onDemandThroughput) {}++ /** The overrides of a restore request. A null member or list means the request did not override it. */+ record RestoreOverrides(String billingMode, Throughput provisionedThroughput,+ OnDemandThroughput onDemandThroughput,+ List<IndexOverride> globalSecondaryIndexes, List<IndexOverride> localSecondaryIndexes,+ List<VectorIndex> vectorIndexes,+ Boolean sseEnabled, String sseType, String kmsMasterKeyId) {}+private final DynamoDbService dynamoDbService;private final DynamoDbStreamService streamService;private final KinesisService kinesisService;@@ -165,6 +180,225 @@ public class NativeDynamoDbTableService {return table;}+ /**+ * Creates the target of a restore in CREATING from {@code source}, a copy of the table the+ * backup holds, with the request's overrides applied. Like AWS, the new table gets the keys,+ * indexes, capacity and encryption, and none of the stream, TTL, point-in-time recovery, tags,+ * deletion protection or table class.+ */+ TableDefinition restoreTable(TableDefinition source, String targetTableName, RestoreOverrides overrides,+ String region) {+ String billingMode = overrides.billingMode() != null+ ? overrides.billingMode()+ : "PAY_PER_REQUEST".equals(source.getBillingMode()) ? "PAY_PER_REQUEST" : "PROVISIONED";+ boolean provisioned = "PROVISIONED".equals(billingMode);+ checkThroughputOverrides(overrides.billingMode(), provisioned, overrides.provisionedThroughput(),+ overrides.onDemandThroughput(), "");+ Throughput throughput = null;+ if (provisioned) {+ throughput = overrides.provisionedThroughput() != null+ ? overrides.provisionedThroughput()+ : new Throughput(source.getProvisionedThroughput().getReadCapacityUnits(),+ source.getProvisionedThroughput().getWriteCapacityUnits());+ }++ List<GlobalSecondaryIndex> gsis = restoredGlobalSecondaryIndexes(source, overrides, provisioned);+ List<LocalSecondaryIndex> lsis = restoredLocalSecondaryIndexes(source, overrides);+ List<DynamoDbService.VectorIndexCreate> vectorIndexes = restoredVectorIndexes(source, overrides);++ Set<String> usedAttributes = new HashSet<>();+ source.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName()));+ gsis.forEach(g -> g.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName())));+ lsis.forEach(l -> l.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName())));+ vectorIndexes.forEach(v -> usedAttributes.addAll(v.index().getSearchSchemaAttributeNames()));+ List<AttributeDefinition> attributeDefinitions = source.getAttributeDefinitions().stream()+ .filter(a -> usedAttributes.contains(a.getAttributeName()))+ .toList();++ Boolean sseEnabled = overrides.sseEnabled() != null ? overrides.sseEnabled() : source.isSseEnabled();+ String sseType = overrides.sseEnabled() != null ? overrides.sseType() : source.getSseType();+ String kmsMasterKeyId = overrides.sseEnabled() != null ? overrides.kmsMasterKeyId() : source.getKmsMasterKeyArn();+ OnDemandThroughput onDemand = provisioned ? null : mergedOnDemand(overrides.onDemandThroughput(),+ source.getOnDemandMaxReadRequestUnits(), source.getOnDemandMaxWriteRequestUnits());+ TableSettings settings = new TableSettings(billingMode, false, null, onDemand, null, null,+ sseEnabled, sseType, kmsMasterKeyId);+ dynamoDbService.requireRestoreTarget(targetTableName, region);+ return createTable(new CreateTableRequest(targetTableName, new ArrayList<>(source.getKeySchema()),+ new ArrayList<>(attributeDefinitions), throughput, gsis, lsis, vectorIndexes, Map.of(), settings),+ "CREATING", region);+ }++ private static List<GlobalSecondaryIndex> restoredGlobalSecondaryIndexes(TableDefinition source,+ RestoreOverrides overrides, boolean provisioned) {+ List<GlobalSecondaryIndex> restored = new ArrayList<>();+ if (overrides.globalSecondaryIndexes() == null) {+ for (GlobalSecondaryIndex gsi : source.getGlobalSecondaryIndexes()) {+ if ("PROVISIONED".equals(overrides.billingMode())) {+ throw invalidRestore("Must override ProvisionedThroughput if BillingMode is overridden to "+ + "PROVISIONED for index " + gsi.getIndexName());+ }+ restored.add(restoredGsi(gsi.getIndexName(), gsi, provisioned, null, null));+ }+ return restored;+ }+ for (IndexOverride override : overrides.globalSecondaryIndexes()) {+ GlobalSecondaryIndex match = source.getGlobalSecondaryIndexes().stream()+ .filter(gsi -> sameIndex(override, gsi.getKeySchema(), gsi.getProjectionType(),+ gsi.getNonKeyAttributes()))+ .findFirst()+ .orElseThrow(() -> indexMismatch(override.indexName()));+ String forIndex = " for index " + override.indexName();+ checkThroughputOverrides(overrides.billingMode(), provisioned, override.provisionedThroughput(),+ override.onDemandThroughput(), forIndex);+ if (provisioned && override.provisionedThroughput() == null) {+ throw invalidRestore("Must specify provisioned throughput" + forIndex);+ }+ restored.add(restoredGsi(override.indexName(), match, provisioned, override.provisionedThroughput(),+ override.onDemandThroughput()));+ }+ return restored;+ }++ /**+ * The table and each index override go through the same checks in this order. An empty+ * OnDemandThroughput is ignored, and the message names the first member the override sets.+ */+ private static void checkThroughputOverrides(String billingModeOverride, boolean provisioned,+ Throughput throughput, OnDemandThroughput onDemand, String suffix) {+ if ("PAY_PER_REQUEST".equals(billingModeOverride) && throughput != null) {+ throw invalidRestore("Cannot override ProvisionedThroughput if BillingMode is overridden to "+ + "PAY_PER_REQUEST" + suffix);+ }+ if ("PROVISIONED".equals(billingModeOverride) && throughput == null) {+ throw invalidRestore("Must override ProvisionedThroughput if BillingMode is overridden to PROVISIONED"+ + suffix);+ }+ if (onDemand != null && (onDemand.maxReadRequestUnits() != null || onDemand.maxWriteRequestUnits() != null)) {+ String cannot = "Cannot override "+ + (onDemand.maxReadRequestUnits() != null ? "MaxReadRequestUnits" : "MaxWriteRequestUnits")+ + " for OnDemandThroughput ";+ if ("PROVISIONED".equals(billingModeOverride)) {+ throw invalidRestore(cannot + "when BillingMode is PROVISIONED" + suffix);+ }+ if (throughput != null) {+ throw invalidRestore(cannot + "when ProvisionedThroughputOverride is present" + suffix);+ }+ if (provisioned) {+ throw invalidRestore(cannot + "unless BillingModeOverride is PAY_PER_REQUEST" + suffix);+ }+ }+ if (!provisioned && throughput != null) {+ throw invalidRestore("Cannot override ProvisionedThroughput unless BillingMode is PROVISIONED" + suffix);+ }+ }++ /** An on-demand override replaces only the members it sets. */+ private static OnDemandThroughput mergedOnDemand(OnDemandThroughput override, Integer maxReadRequestUnits,+ Integer maxWriteRequestUnits) {+ Integer maxRead = override != null && override.maxReadRequestUnits() != null+ ? override.maxReadRequestUnits() : maxReadRequestUnits;+ Integer maxWrite = override != null && override.maxWriteRequestUnits() != null+ ? override.maxWriteRequestUnits() : maxWriteRequestUnits;+ return maxRead == null && maxWrite == null ? null : new OnDemandThroughput(maxRead, maxWrite);+ }++ private static GlobalSecondaryIndex restoredGsi(String indexName, GlobalSecondaryIndex source, boolean provisioned,+ Throughput throughput, OnDemandThroughput onDemandOverride) {+ GlobalSecondaryIndex gsi = new GlobalSecondaryIndex(indexName, new ArrayList<>(source.getKeySchema()), null,+ source.getProjectionType(), new ArrayList<>(source.getNonKeyAttributes()));+ if (provisioned) {+ gsi.getProvisionedThroughput().setReadCapacityUnits(throughput != null ? throughput.readCapacityUnits()+ : source.getProvisionedThroughput().getReadCapacityUnits());+ gsi.getProvisionedThroughput().setWriteCapacityUnits(throughput != null ? throughput.writeCapacityUnits()+ : source.getProvisionedThroughput().getWriteCapacityUnits());+ return gsi;+ }+ OnDemandThroughput onDemand = mergedOnDemand(onDemandOverride, source.getOnDemandMaxReadRequestUnits(),+ source.getOnDemandMaxWriteRequestUnits());+ if (onDemand != null) {+ gsi.setOnDemandMaxReadRequestUnits(onDemand.maxReadRequestUnits());+ gsi.setOnDemandMaxWriteRequestUnits(onDemand.maxWriteRequestUnits());+ }+ return gsi;+ }++ private static List<LocalSecondaryIndex> restoredLocalSecondaryIndexes(TableDefinition source,+ RestoreOverrides overrides) {+ List<LocalSecondaryIndex> restored = new ArrayList<>();+ if (overrides.localSecondaryIndexes() == null) {+ for (LocalSecondaryIndex lsi : source.getLocalSecondaryIndexes()) {+ restored.add(new LocalSecondaryIndex(lsi.getIndexName(), new ArrayList<>(lsi.getKeySchema()), null,+ lsi.getProjectionType(), new ArrayList<>(lsi.getNonKeyAttributes())));+ }+ return restored;+ }+ for (IndexOverride override : overrides.localSecondaryIndexes()) {+ LocalSecondaryIndex match = source.getLocalSecondaryIndexes().stream()+ .filter(lsi -> sameIndex(override, lsi.getKeySchema(), lsi.getProjectionType(),+ lsi.getNonKeyAttributes()))+ .findFirst()+ .orElseThrow(() -> indexMismatch(override.indexName()));+ restored.add(new LocalSecondaryIndex(override.indexName(), new ArrayList<>(match.getKeySchema()), null,+ match.getProjectionType(), new ArrayList<>(match.getNonKeyAttributes())));+ }+ return restored;+ }++ /** Vector indexes, unlike the others, are matched by name. */+ private static List<DynamoDbService.VectorIndexCreate> restoredVectorIndexes(TableDefinition source,+ RestoreOverrides overrides) {+ List<VectorIndex> kept = new ArrayList<>();+ if (overrides.vectorIndexes() == null) {+ kept.addAll(source.getVectorIndexes());+ } else {+ for (VectorIndex override : overrides.vectorIndexes()) {+ kept.add(source.getVectorIndexes().stream()+ .filter(v -> v.getIndexName().equals(override.getIndexName()))+ .findFirst()+ .orElseThrow(() -> invalidRestore("Vector index " + override.getIndexName()+ + " does not match a vector index that existed in the source table and cannot be "+ + "created during the restore operation")));+ }+ }+ List<DynamoDbService.VectorIndexCreate> restored = new ArrayList<>();+ for (int i = 0; i < kept.size(); i++) {+ VectorIndex v = kept.get(i);+ VectorIndex copy = new VectorIndex(v.getIndexName(), v.getVectorAttributeName(),+ new ArrayList<>(v.getSearchSchema()), v.getProjectionType(),+ new ArrayList<>(v.getNonKeyAttributes()), v.getDimensions(), v.getDistanceFunction());+ restored.add(new DynamoDbService.VectorIndexCreate(copy, "vectorIndexOverride." + (i + 1) + ".member"));+ }+ return restored;+ }++ private static boolean sameIndex(IndexOverride override, List<KeySchemaElement> keySchema, String projectionType,+ List<String> nonKeyAttributes) {+ if (override.keySchema().size() != keySchema.size()) {+ return false;+ }+ for (int i = 0; i < keySchema.size(); i++) {+ KeySchemaElement want = override.keySchema().get(i);+ KeySchemaElement have = keySchema.get(i);+ if (!want.getAttributeName().equals(have.getAttributeName()) || !want.getKeyType().equals(have.getKeyType())) {+ return false;+ }+ }+ String type = projectionType != null ? projectionType : "ALL";+ if (!type.equals(override.projectionType())) {+ return false;+ }+ return !"INCLUDE".equals(type) || Set.copyOf(nonKeyAttributes).equals(Set.copyOf(override.nonKeyAttributes()));+ }++ private static AwsException indexMismatch(String indexName) {+ return invalidRestore("Index " + indexName + " does not match a secondary index that existed in the source "+ + "table and cannot be created during the restore operation");+ }++ private static AwsException invalidRestore(String message) {+ return new AwsException("ValidationException", "Invalid Request: " + message, 400);+ }+/*** Updates the table, applies the request's settings and replica changes to it and persists* the result.diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.javanew file mode 100644index 0000000000..6ed037daf7--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.java@@ -0,0 +1,59 @@+package io.github.hectorvent.floci.services.dynamodb.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.databind.JsonNode;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.time.Instant;+import java.util.ArrayList;+import java.util.List;++/**+ * An on-demand backup: its details, the source table as it was when the backup was taken, and+ * the items it holds.+ */+@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+public class TableBackup {++ private String backupArn;+ private String backupName;+ private String backupType;+ private Instant creationDateTime;+ private Instant expiryDateTime;+ private long sizeBytes;+ private String region;+ private TableDefinition table;+ private List<JsonNode> items;++ public TableBackup() {+ this.items = new ArrayList<>();+ }++ public String getBackupArn() { return backupArn; }+ public void setBackupArn(String backupArn) { this.backupArn = backupArn; }++ public String getBackupName() { return backupName; }+ public void setBackupName(String backupName) { this.backupName = backupName; }++ public String getBackupType() { return backupType; }+ public void setBackupType(String backupType) { this.backupType = backupType; }++ public Instant getCreationDateTime() { return creationDateTime; }+ public void setCreationDateTime(Instant creationDateTime) { this.creationDateTime = creationDateTime; }++ public Instant getExpiryDateTime() { return expiryDateTime; }+ public void setExpiryDateTime(Instant expiryDateTime) { this.expiryDateTime = expiryDateTime; }++ public long getSizeBytes() { return sizeBytes; }+ public void setSizeBytes(long sizeBytes) { this.sizeBytes = sizeBytes; }++ public String getRegion() { return region; }+ public void setRegion(String region) { this.region = region; }++ public TableDefinition getTable() { return table; }+ public void setTable(TableDefinition table) { this.table = table; }++ public List<JsonNode> getItems() { return items; }+ public void setItems(List<JsonNode> items) { this.items = items != null ? items : new ArrayList<>(); }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.javanew file mode 100644index 0000000000..4a8578cfd6--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java@@ -0,0 +1,773 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.DeserializationFeature;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.math.BigDecimal;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class DynamoDbBackupIntegrationTest {++ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";+ // JsonPath reads numbers as float, which drops whole seconds at epoch magnitude.+ private static final ObjectMapper EXACT = new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);+ private static final String TABLE_ARN_PREFIX = "arn:aws:dynamodb:us-east-1:000000000000:table/";+ private static final String BACKUP_ARN_PATTERN =+ "arn:aws:dynamodb:us-east-1:000000000000:table/%s/backup/\\d{14}-[0-9a-f]{8}";+ private static final String OTHER_REGION_AUTH =+ "AWS4-HMAC-SHA256 Credential=AKID/20260215/eu-west-2/dynamodb/aws4_request, "+ + "SignedHeaders=host, Signature=abc";+ private static final String OTHER_ACCOUNT_AUTH =+ "AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/dynamodb/aws4_request, "+ + "SignedHeaders=host, Signature=abc";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void createBackupAnswersCreatingAndDescribeBackupAnswersAvailable() {+ String table = "BackupIT-describe";+ createIndexedTable(table);+ putItem(table, "a", "1");+ ddb("UpdateTimeToLive", """+ {"TableName": "%s", "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "ttl"}}+ """.formatted(table)).statusCode(200);++ String backupArn = ddb("CreateBackup", """+ {"TableName": "%s", "BackupName": "describe_me"}+ """.formatted(table))+ .statusCode(200)+ .body("BackupDetails.BackupName", equalTo("describe_me"))+ .body("BackupDetails.BackupStatus", equalTo("CREATING"))+ .body("BackupDetails.BackupType", equalTo("USER"))+ .body("BackupDetails.BackupSizeBytes", equalTo(0))+ .body("BackupDetails.BackupCreationDateTime", notNullValue())+ .body("BackupDetails", not(hasKey("BackupExpiryDateTime")))+ .extract().path("BackupDetails.BackupArn");+ assertTrue(backupArn.matches(BACKUP_ARN_PATTERN.formatted(table)), backupArn);++ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")+ .statusCode(200)+ .body("BackupDescription.BackupDetails.BackupArn", equalTo(backupArn))+ .body("BackupDescription.BackupDetails.BackupStatus", equalTo("AVAILABLE"))+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table))+ .body("BackupDescription.SourceTableDetails.TableArn", equalTo(TABLE_ARN_PREFIX + table))+ .body("BackupDescription.SourceTableDetails.TableId", notNullValue())+ .body("BackupDescription.SourceTableDetails.TableCreationDateTime", notNullValue())+ .body("BackupDescription.SourceTableDetails.ItemCount", equalTo(1))+ .body("BackupDescription.SourceTableDetails.BillingMode", equalTo("PAY_PER_REQUEST"))+ .body("BackupDescription.SourceTableDetails.KeySchema.AttributeName", contains("pk", "sk"))+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput.ReadCapacityUnits", equalTo(0))+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput",+ not(hasKey("NumberOfDecreasesToday")))+ .body("BackupDescription.SourceTableFeatureDetails.GlobalSecondaryIndexes[0].IndexName",+ equalTo("gsi1"))+ .body("BackupDescription.SourceTableFeatureDetails.GlobalSecondaryIndexes[0]",+ not(hasKey("IndexStatus")))+ .body("BackupDescription.SourceTableFeatureDetails.LocalSecondaryIndexes[0].IndexName",+ equalTo("lsi1"))+ .body("BackupDescription.SourceTableFeatureDetails.LocalSecondaryIndexes[0]",+ not(hasKey("IndexArn")))+ .body("BackupDescription.SourceTableFeatureDetails.StreamDescription.StreamEnabled", equalTo(true))+ .body("BackupDescription.SourceTableFeatureDetails.StreamDescription.StreamViewType",+ equalTo("NEW_AND_OLD_IMAGES"))+ .body("BackupDescription.SourceTableFeatureDetails.TimeToLiveDescription.AttributeName",+ equalTo("ttl"))+ .body("BackupDescription.SourceTableFeatureDetails.TimeToLiveDescription.TimeToLiveStatus",+ equalTo("ENABLED"))+ .body("BackupDescription.SourceTableFeatureDetails.SSEDescription.SSEType", equalTo("KMS"));+ }++ @Test+ void plainTableBackupHasNoFeatureDetails() {+ String table = "BackupIT-plain";+ createPlainTable(table);+ String backupArn = createBackup(table, "plain");++ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")+ .statusCode(200)+ .body("BackupDescription.SourceTableDetails.BillingMode", equalTo("PROVISIONED"))+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput.ReadCapacityUnits", equalTo(1))+ .body("BackupDescription.SourceTableFeatureDetails.size()", equalTo(0));+ }++ @Test+ void createBackupOfMissingTableIsTableNotFound() {+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"abc\"}")+ .statusCode(400)+ .body("__type", containsString("TableNotFoundException"))+ .body("message", equalTo("Table not found: BackupIT-missing"));+ }++ @Test+ void createBackupValidatesItsMembers() {+ ddb("CreateBackup", "{\"BackupName\": \"abc\"}")+ .statusCode(400)+ .body("message", equalTo("The parameter 'TableName' is required but was not present in the request"));+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\"}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value null at 'backupName' failed to "+ + "satisfy constraint: Member must not be null"));+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"ab\"}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value 'ab' at 'backupName' failed to "+ + "satisfy constraint: Member must have length greater than or equal to 3"));+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"a b\"}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value 'a b' at 'backupName' failed to "+ + "satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));+ ddb("CreateBackup", "{\"TableName\": \"a b\", \"BackupName\": \"c d\"}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value 'a b' at 'tableName' failed to "+ + "satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));+ }++ @Test+ void backupArnsAreCheckedInTwoSteps() {+ String base = TABLE_ARN_PREFIX + "BackupIT-none/backup/";+ for (String arn : List.of("x".repeat(40), base, base + "xy", base + "01700000000000-abcdef12/extra")) {+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")+ .statusCode(400)+ .body("__type", containsString("ValidationException"))+ .body("message", equalTo("Invalid Backup ARN"));+ }+ for (String arn : List.of(base + "xyz", base + "1700000000000-abcdef12", base + "01700000000000-abcdefgh",+ "arn:aws:s3:us-east-1:000000000000:table/BackupIT-none/backup/01700000000000-abcdef12",+ "arn:aws:dynamodb:nowhere-1:000000000000:table/BackupIT-none/backup/01700000000000-abcdef12")) {+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: BackupArn is not valid"));+ }+ ddb("DeleteBackup", "{}")+ .statusCode(400)+ .body("message", equalTo("Invalid Backup ARN"));+ }++ @Test+ void unknownBackupIsBackupNotFound() {+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-ABCDEF12";+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"))+ .body("message", equalTo("Backup not found: " + arn));+ ddb("DeleteBackup", "{\"BackupArn\": \"" + arn + "\"}")+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"));+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + arn + "\", \"TargetTableName\": \"BackupIT-describe\"}")+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"));+ }++ @Test+ void backupIsScopedToItsRegionAndAccount() {+ String table = "BackupIT-scope";+ createPlainTable(table);+ String backupArn = createBackup(table, "scoped");++ given().header("Authorization", OTHER_REGION_AUTH)+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeBackup")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"BackupArn\": \"" + backupArn + "\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"));+ given().header("Authorization", OTHER_REGION_AUTH)+ .header("X-Amz-Target", "DynamoDB_20120810.ListBackups")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableName\": \"" + table + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("BackupSummaries", empty());+ given().header("Authorization", OTHER_ACCOUNT_AUTH)+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeBackup")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"BackupArn\": \"" + backupArn + "\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("AccessDeniedException"))+ .body("message", equalTo("Access is denied"));+ }++ @Test+ void listBackupsPagesInCreationOrder() {+ String table = "BackupIT-list";+ createPlainTable(table);+ String first = createBackup(table, "same_name");+ String second = createBackup(table, "same_name");+ String third = createBackup(table, "third");++ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(first, second, third))+ .body("BackupSummaries[0].BackupStatus", equalTo("AVAILABLE"))+ .body("BackupSummaries[0].TableName", equalTo(table))+ .body("BackupSummaries[0].TableArn", equalTo(TABLE_ARN_PREFIX + table))+ .body("BackupSummaries[0].TableId", notNullValue())+ .body("$", not(hasKey("LastEvaluatedBackupArn")));+ ddb("ListBackups", "{\"TableName\": \"" + TABLE_ARN_PREFIX + table + "\", \"Limit\": 2}")+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(first, second))+ .body("LastEvaluatedBackupArn", equalTo(second));+ ddb("ListBackups", """+ {"TableName": "%s", "Limit": 2, "ExclusiveStartBackupArn": "%s"}+ """.formatted(table, second))+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(third))+ .body("$", not(hasKey("LastEvaluatedBackupArn")));+ ddb("ListBackups", """+ {"TableName": "%s", "ExclusiveStartBackupArn": "%s"}+ """.formatted(table, TABLE_ARN_PREFIX + table + "/backup/01700000000000-abcdef12"))+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(first, second, third));+ }++ @Test+ void listBackupsFiltersByTableTypeAndTime() throws Exception {+ String table = "BackupIT-filter";+ createPlainTable(table);+ String backupArn = createBackup(table, "filtered");+ BigDecimal created = decimalAt(ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}"),+ "/BackupDescription/BackupDetails/BackupCreationDateTime");++ ddb("ListBackups", "{\"TableName\": \"BackupIT-nosuch\"}")+ .statusCode(200)+ .body("BackupSummaries", empty());+ ddb("ListBackups", "{\"TableName\": \"" + table + "\", \"BackupType\": \"SYSTEM\"}")+ .statusCode(200)+ .body("BackupSummaries", empty());+ ddb("ListBackups", "{\"TableName\": \"" + table + "\", \"BackupType\": \"ALL\"}")+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(backupArn));+ ddb("ListBackups", """+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}+ """.formatted(table, created, created.add(BigDecimal.ONE)))+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(backupArn));+ ddb("ListBackups", """+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}+ """.formatted(table, created.subtract(BigDecimal.ONE), created))+ .statusCode(200)+ .body("BackupSummaries", empty());+ ddb("ListBackups", """+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}+ """.formatted(table, created, created.subtract(BigDecimal.ONE)))+ .statusCode(400)+ .body("message", equalTo("One or more parameter values were invalid: "+ + "Time range lower bound must be less than or equal to upper bound"));+ }++ @Test+ void listBackupsReportsEveryViolation() {+ ddb("ListBackups", "{\"Limit\": 0, \"BackupType\": \"BAD\"}")+ .statusCode(400)+ .body("message", equalTo("2 validation errors detected: Value '0' at 'limit' failed to satisfy "+ + "constraint: Member must have value greater than or equal to 1; Value 'BAD' at "+ + "'backupType' failed to satisfy constraint: Member must satisfy enum value set: "+ + "[ALL, AWS_BACKUP, USER, SYSTEM]"));+ ddb("ListBackups", "{\"Limit\": 101}")+ .statusCode(400)+ .body("message", equalTo("1 validation error detected: Value '101' at 'limit' failed to satisfy "+ + "constraint: Member must have value less than or equal to 100"));+ ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + "x".repeat(40) + "\"}")+ .statusCode(400)+ .body("message", equalTo("Invalid Backup ARN"));+ }++ @Test+ void deleteBackupAnswersDeletedAndTheBackupIsGone() {+ String table = "BackupIT-delete";+ createPlainTable(table);+ String backupArn = createBackup(table, "doomed");++ ddb("DeleteBackup", "{\"BackupArn\": \"" + backupArn + "\"}")+ .statusCode(200)+ .body("BackupDescription.BackupDetails.BackupArn", equalTo(backupArn))+ .body("BackupDescription.BackupDetails.BackupStatus", equalTo("DELETED"))+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table));+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")+ .statusCode(400)+ .body("__type", containsString("BackupNotFoundException"));+ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")+ .statusCode(200)+ .body("BackupSummaries", empty());+ }++ @Test+ void backupOutlivesItsTable() {+ String table = "BackupIT-outlive";+ createPlainTable(table);+ putItem(table, "kept");+ String backupArn = createBackup(table, "survivor");+ ddb("DeleteTable", "{\"TableName\": \"" + table + "\"}").statusCode(200);++ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")+ .statusCode(200)+ .body("BackupSummaries.BackupArn", contains(backupArn));+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")+ .statusCode(200)+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table));+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + table + "\"}")+ .statusCode(200);+ ddb("GetItem", "{\"TableName\": \"" + table + "\", \"Key\": {\"pk\": {\"S\": \"kept\"}}}")+ .statusCode(200)+ .body("Item.pk.S", equalTo("kept"));+ }++ @Test+ void restoreTableFromBackupCopiesTheBackedUpItemsAndSchema() throws Exception {+ String table = "BackupIT-restore-src";+ String target = "BackupIT-restore-dst";+ createIndexedTable(table);+ ddb("UpdateTimeToLive", """+ {"TableName": "%s", "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "ttl"}}+ """.formatted(table)).statusCode(200);+ ddb("UpdateContinuousBackups", """+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}+ """.formatted(table)).statusCode(200);+ putItem(table, "a", "1");+ putItem(table, "a", "2");+ String backupArn = createBackup(table, "restore_me");+ putItem(table, "a", "after-backup");+ BigDecimal backupCreated = decimalAt(ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}"),+ "/BackupDescription/BackupDetails/BackupCreationDateTime");++ ValidatableResponse restored = ddb("RestoreTableFromBackup",+ "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + target + "\"}");+ assertEquals(0, backupCreated.compareTo(decimalAt(restored, "/TableDescription/RestoreSummary/RestoreDateTime")));+ restored.statusCode(200)+ .body("TableDescription.TableName", equalTo(target))+ .body("TableDescription.TableArn", equalTo(TABLE_ARN_PREFIX + target))+ .body("TableDescription.TableStatus", equalTo("CREATING"))+ .body("TableDescription.RestoreSummary.SourceBackupArn", equalTo(backupArn))+ .body("TableDescription.RestoreSummary.SourceTableArn", equalTo(TABLE_ARN_PREFIX + table))+ .body("TableDescription.RestoreSummary.RestoreInProgress", equalTo(true))+ .body("TableDescription.GlobalSecondaryIndexes[0].IndexArn",+ equalTo(TABLE_ARN_PREFIX + target + "/index/gsi1"))+ .body("TableDescription.LocalSecondaryIndexes[0].IndexName", equalTo("lsi1"))+ .body("TableDescription.SSEDescription.SSEType", equalTo("KMS"));++ ddb("DescribeTable", "{\"TableName\": \"" + target + "\"}")+ .statusCode(200)+ .body("Table.TableStatus", equalTo("ACTIVE"))+ .body("Table", not(hasKey("RestoreSummary")))+ .body("Table", not(hasKey("StreamSpecification")))+ .body("Table.DeletionProtectionEnabled", equalTo(false))+ .body("Table.TableClassSummary.TableClass", equalTo("STANDARD"))+ .body("Table.BillingModeSummary.BillingMode", equalTo("PAY_PER_REQUEST"))+ .body("Table.ItemCount", equalTo(2));+ ddb("DescribeTimeToLive", "{\"TableName\": \"" + target + "\"}")+ .statusCode(200)+ .body("TimeToLiveDescription.TimeToLiveStatus", equalTo("DISABLED"));+ ddb("DescribeContinuousBackups", "{\"TableName\": \"" + target + "\"}")+ .statusCode(200)+ .body("ContinuousBackupsDescription.PointInTimeRecoveryDescription.PointInTimeRecoveryStatus",+ equalTo("DISABLED"));+ ddb("ListTagsOfResource", "{\"ResourceArn\": \"" + TABLE_ARN_PREFIX + target + "\"}")+ .statusCode(200)+ .body("Tags", empty());+ ddb("Scan", "{\"TableName\": \"" + target + "\"}")+ .statusCode(200)+ .body("Items.sk.S", containsInAnyOrder("1", "2"));+ ddb("Query", """+ {"TableName": "%s", "IndexName": "gsi1", "KeyConditionExpression": "g = :g",+ "ExpressionAttributeValues": {":g": {"S": "g-1"}}}+ """.formatted(target))+ .statusCode(200)+ .body("Count", equalTo(1));+ }++ @Test+ void restoreTableFromBackupRefusesAnExistingTarget() {+ String table = "BackupIT-exists";+ createPlainTable(table);+ String backupArn = createBackup(table, "exists");++ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + table + "\"}")+ .statusCode(400)+ .body("__type", containsString("TableAlreadyExistsException"))+ .body("message", equalTo("Table already exists: " + table));+ }++ @Test+ void restoreTableFromBackupChecksTheArnFirst() {+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + "x".repeat(40) + "\", \"TargetTableName\": \"a b\"}")+ .statusCode(400)+ .body("message", equalTo("Invalid Backup ARN"));+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + arn + "\"}")+ .statusCode(400)+ .body("message", equalTo("The parameter 'TableName' is required but was not present in the request"));+ ddb("RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "a b", "BillingModeOverride": "BAD"}+ """.formatted(arn))+ .statusCode(400)+ .body("message", equalTo("2 validation errors detected: Value 'BAD' at 'billingModeOverride' "+ + "failed to satisfy constraint: Member must satisfy enum value set: "+ + "[PROVISIONED, PAY_PER_REQUEST]; Value 'a b' at 'targetTableName' failed to satisfy "+ + "constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));+ }++ @Test+ void restoreReportsEveryViolationBeforeTheBackupLookup() {+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";+ String notNull = "failed to satisfy constraint: Member must not be null";+ String atLeastOne = "failed to satisfy constraint: Member must have value greater than or equal to 1";+ ddb("RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "bad name!", "BillingModeOverride": "BAD",+ "ProvisionedThroughputOverride": {},+ "GlobalSecondaryIndexOverride": [%s]}+ """.formatted(arn, gsi1(", \"ProvisionedThroughput\": {}").replace("\"gsi1\"", "\"x!\"")))+ .statusCode(400)+ .body("message", equalTo("8 validation errors detected: "+ + "Value null at 'provisionedThroughputOverride.writeCapacityUnits' " + notNull + "; "+ + "Value null at 'provisionedThroughputOverride.readCapacityUnits' " + notNull + "; "+ + "Value 'BAD' at 'billingModeOverride' failed to satisfy constraint: Member must satisfy "+ + "enum value set: [PROVISIONED, PAY_PER_REQUEST]; "+ + "Value 'bad name!' at 'targetTableName' failed to satisfy constraint: Member must satisfy "+ + "regular expression pattern: [a-zA-Z0-9_.-]+; "+ + "Value 'x!' at 'globalSecondaryIndexOverride.1.member.indexName' failed to satisfy "+ + "constraint: Member must have length greater than or equal to 3; "+ + "Value 'x!' at 'globalSecondaryIndexOverride.1.member.indexName' failed to satisfy "+ + "constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+; "+ + "Value null at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."+ + "writeCapacityUnits' " + notNull + "; "+ + "Value null at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."+ + "readCapacityUnits' " + notNull));+ ddb("RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "BackupIT-none-x", "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 0}}+ """.formatted(arn))+ .statusCode(400)+ .body("message", equalTo("2 validation errors detected: "+ + "Value null at 'provisionedThroughputOverride.writeCapacityUnits' " + notNull + "; "+ + "Value '0' at 'provisionedThroughputOverride.readCapacityUnits' " + atLeastOne));+ ddb("RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "BackupIT-none-x", "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 0, "WriteCapacityUnits": 1},+ "GlobalSecondaryIndexOverride": [%s]}+ """.formatted(arn, gsi1(", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 0, "+ + "\"WriteCapacityUnits\": 0}")))+ .statusCode(400)+ .body("message", equalTo("3 validation errors detected: "+ + "Value '0' at 'provisionedThroughputOverride.readCapacityUnits' " + atLeastOne + "; "+ + "Value '0' at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."+ + "writeCapacityUnits' " + atLeastOne + "; "+ + "Value '0' at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."+ + "readCapacityUnits' " + atLeastOne));+ }++ @Test+ void restoreOverridesFollowTheBillingMode() {+ String table = "BackupIT-billing";+ createIndexedTable(table);+ String backupArn = createBackup(table, "billing");++ restore(backupArn, "BackupIT-billing-x", "\"BillingModeOverride\": \"PROVISIONED\"")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Must override ProvisionedThroughput if BillingMode is "+ + "overridden to PROVISIONED"));+ restore(backupArn, "BackupIT-billing-x", """+ "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3}+ """)+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Must override ProvisionedThroughput if BillingMode is "+ + "overridden to PROVISIONED for index gsi1"));+ restore(backupArn, "BackupIT-billing-x",+ "\"ProvisionedThroughputOverride\": {\"ReadCapacityUnits\": 2, \"WriteCapacityUnits\": 3}")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput unless "+ + "BillingMode is PROVISIONED"));+ restore(backupArn, "BackupIT-billing-x", """+ "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10}+ """)+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "+ + "OnDemandThroughput when BillingMode is PROVISIONED"));+ restore(backupArn, "BackupIT-billing-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(+ ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 4, \"WriteCapacityUnits\": 5}") + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput unless "+ + "BillingMode is PROVISIONED for index gsi1"));++ restore(backupArn, "BackupIT-billing-provisioned", """+ "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},+ "GlobalSecondaryIndexOverride": [%s]+ """.formatted(gsi1(", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 4, \"WriteCapacityUnits\": 5}")))+ .statusCode(200)+ .body("TableDescription", not(hasKey("BillingModeSummary")))+ .body("TableDescription.ProvisionedThroughput.ReadCapacityUnits", equalTo(2))+ .body("TableDescription.ProvisionedThroughput.WriteCapacityUnits", equalTo(3))+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",+ equalTo(4));+ restore(backupArn, "BackupIT-billing-ondemand", "\"OnDemandThroughputOverride\": {\"MaxReadRequestUnits\": 10}")+ .statusCode(200)+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(10));+ }++ @Test+ void restoreOnDemandOverridesMergeWithTheSourceAndFollowTheBillingMode() {+ String table = "BackupIT-ondemand";+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "g", "AttributeType": "S"}],+ "GlobalSecondaryIndexes": [%s],+ "BillingMode": "PAY_PER_REQUEST",+ "OnDemandThroughput": {"MaxReadRequestUnits": 7, "MaxWriteRequestUnits": 8}+ }+ """.formatted(table, gsi1(", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5, "+ + "\"MaxWriteRequestUnits\": 6}"))).statusCode(200);+ String backupArn = createBackup(table, "ondemand");+ String gsiThroughput = ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 1, \"WriteCapacityUnits\": 1}";+ String provisioned = """+ "BillingModeOverride": "PROVISIONED",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},+ """;++ restore(backupArn, "BackupIT-ondemand-x", provisioned + "\"GlobalSecondaryIndexOverride\": [" + gsi1(+ gsiThroughput + ", \"OnDemandThroughput\": {\"MaxWriteRequestUnits\": 5}") + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxWriteRequestUnits for "+ + "OnDemandThroughput when BillingMode is PROVISIONED for index gsi1"));+ restore(backupArn, "BackupIT-ondemand-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(+ gsiThroughput + ", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}") + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "+ + "OnDemandThroughput when ProvisionedThroughputOverride is present for index gsi1"));+ restore(backupArn, "BackupIT-ondemand-x", """+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10}+ """)+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "+ + "OnDemandThroughput when ProvisionedThroughputOverride is present"));+ restore(backupArn, "BackupIT-ondemand-x", """+ "BillingModeOverride": "PAY_PER_REQUEST",+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3}+ """)+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput if BillingMode "+ + "is overridden to PAY_PER_REQUEST"));+ restore(backupArn, "BackupIT-ondemand-x", "\"BillingModeOverride\": \"PAY_PER_REQUEST\", "+ + "\"GlobalSecondaryIndexOverride\": [" + gsi1(gsiThroughput) + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput if BillingMode "+ + "is overridden to PAY_PER_REQUEST for index gsi1"));++ restore(backupArn, "BackupIT-ondemand-provisioned",+ provisioned + "\"GlobalSecondaryIndexOverride\": [" + gsi1(gsiThroughput) + "]")+ .statusCode(200)+ .body("TableDescription", not(hasKey("OnDemandThroughput")))+ .body("TableDescription.GlobalSecondaryIndexes[0]", not(hasKey("OnDemandThroughput")));+ restore(backupArn, "BackupIT-ondemand-merged", """+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10},+ "GlobalSecondaryIndexOverride": [%s]+ """.formatted(gsi1(", \"OnDemandThroughput\": {\"MaxWriteRequestUnits\": 9}")))+ .statusCode(200)+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(10))+ .body("TableDescription.OnDemandThroughput.MaxWriteRequestUnits", equalTo(8))+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxReadRequestUnits", equalTo(5))+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxWriteRequestUnits",+ equalTo(9));+ restore(backupArn, "BackupIT-ondemand-empty", """+ "OnDemandThroughputOverride": {},+ "GlobalSecondaryIndexOverride": [%s]+ """.formatted(gsi1(", \"OnDemandThroughput\": {}")))+ .statusCode(200)+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(7))+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxWriteRequestUnits",+ equalTo(6));+ }++ @Test+ void restoreOfAProvisionedTableNeedsThroughputForEveryIndexOverride() {+ String table = "BackupIT-provisioned";+ String gsiThroughput = ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 1, \"WriteCapacityUnits\": 1}";+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "g", "AttributeType": "S"}],+ "GlobalSecondaryIndexes": [%s],+ "ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}+ }+ """.formatted(table, gsi1(gsiThroughput))).statusCode(200);+ String backupArn = createBackup(table, "provisioned");++ restore(backupArn, "BackupIT-provisioned-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1("") + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Must specify provisioned throughput for index gsi1"));+ restore(backupArn, "BackupIT-provisioned-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(+ ", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}") + "]")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "+ + "OnDemandThroughput unless BillingModeOverride is PAY_PER_REQUEST for index gsi1"));+ restore(backupArn, "BackupIT-provisioned-x", "\"OnDemandThroughputOverride\": {\"MaxReadRequestUnits\": 10}")+ .statusCode(400)+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "+ + "OnDemandThroughput unless BillingModeOverride is PAY_PER_REQUEST"));++ restore(backupArn, "BackupIT-provisioned-kept", """+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},+ "OnDemandThroughputOverride": {},+ "GlobalSecondaryIndexOverride": [%s]+ """.formatted(gsi1(gsiThroughput + ", \"OnDemandThroughput\": {}")))+ .statusCode(200)+ .body("TableDescription.ProvisionedThroughput.ReadCapacityUnits", equalTo(2))+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",+ equalTo(1))+ .body("TableDescription.GlobalSecondaryIndexes[0]", not(hasKey("OnDemandThroughput")));+ restore(backupArn, "BackupIT-provisioned-ondemand", """+ "BillingModeOverride": "PAY_PER_REQUEST",+ "GlobalSecondaryIndexOverride": [%s]+ """.formatted(gsi1(", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}")))+ .statusCode(200)+ .body("TableDescription.BillingModeSummary.BillingMode", equalTo("PAY_PER_REQUEST"))+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",+ equalTo(0))+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxReadRequestUnits", equalTo(5))+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput",+ not(hasKey("MaxWriteRequestUnits")));+ }++ @Test+ void restoreIndexOverridesMustMatchTheSourceIndexes() {+ String table = "BackupIT-indexes";+ createIndexedTable(table);+ String backupArn = createBackup(table, "indexes");+ String mismatch = "Invalid Request: Index %s does not match a secondary index that existed in the source "+ + "table and cannot be created during the restore operation";++ restore(backupArn, "BackupIT-indexes-x", """+ "GlobalSecondaryIndexOverride": [{"IndexName": "gsi1",+ "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], "Projection": {"ProjectionType": "KEYS_ONLY"}}]+ """)+ .statusCode(400)+ .body("message", equalTo(mismatch.formatted("gsi1")));+ restore(backupArn, "BackupIT-indexes-x", """+ "LocalSecondaryIndexOverride": [{"IndexName": "lsi1",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],+ "Projection": {"ProjectionType": "ALL"}}]+ """)+ .statusCode(400)+ .body("message", equalTo(mismatch.formatted("lsi1")));++ restore(backupArn, "BackupIT-indexes-renamed", """+ "LocalSecondaryIndexOverride": [{"IndexName": "renamed",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],+ "Projection": {"ProjectionType": "KEYS_ONLY"}}]+ """)+ .statusCode(200)+ .body("TableDescription.LocalSecondaryIndexes.IndexName", contains("renamed"))+ .body("TableDescription.GlobalSecondaryIndexes.IndexName", contains("gsi1"));+ restore(backupArn, "BackupIT-indexes-dropped", """+ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [],+ "SSESpecificationOverride": {"Enabled": false}+ """)+ .statusCode(200)+ .body("TableDescription", not(hasKey("GlobalSecondaryIndexes")))+ .body("TableDescription", not(hasKey("LocalSecondaryIndexes")))+ .body("TableDescription", not(hasKey("SSEDescription")))+ .body("TableDescription.AttributeDefinitions.AttributeName", containsInAnyOrder("pk", "sk"));+ }++ private void createIndexedTable(String table) {+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}],+ "AttributeDefinitions": [+ {"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "sk", "AttributeType": "S"},+ {"AttributeName": "g", "AttributeType": "S"},+ {"AttributeName": "l", "AttributeType": "S"}+ ],+ "GlobalSecondaryIndexes": [%s],+ "LocalSecondaryIndexes": [{"IndexName": "lsi1",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],+ "Projection": {"ProjectionType": "KEYS_ONLY"}}],+ "BillingMode": "PAY_PER_REQUEST",+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"},+ "SSESpecification": {"Enabled": true, "SSEType": "KMS"},+ "TableClass": "STANDARD_INFREQUENT_ACCESS",+ "DeletionProtectionEnabled": true,+ "Tags": [{"Key": "k", "Value": "v"}]+ }+ """.formatted(table, gsi1(""))).statusCode(200);+ }++ private static String gsi1(String extra) {+ return "{\"IndexName\": \"gsi1\", \"KeySchema\": [{\"AttributeName\": \"g\", \"KeyType\": \"HASH\"}], "+ + "\"Projection\": {\"ProjectionType\": \"ALL\"}" + extra + "}";+ }++ private void createPlainTable(String table) {+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],+ "ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}+ }+ """.formatted(table)).statusCode(200);+ }++ private void putItem(String table, String pk) {+ ddb("PutItem", "{\"TableName\": \"" + table + "\", \"Item\": {\"pk\": {\"S\": \"" + pk + "\"}}}")+ .statusCode(200);+ }++ private void putItem(String table, String pk, String sk) {+ ddb("PutItem", """+ {"TableName": "%s", "Item": {"pk": {"S": "%s"}, "sk": {"S": "%s"}, "g": {"S": "g-%s"},+ "l": {"S": "l-%s"}, "ttl": {"N": "4102444800"}}}+ """.formatted(table, pk, sk, sk, sk)).statusCode(200);+ }++ private String createBackup(String table, String backupName) {+ return ddb("CreateBackup", "{\"TableName\": \"" + table + "\", \"BackupName\": \"" + backupName + "\"}")+ .statusCode(200)+ .extract().path("BackupDetails.BackupArn");+ }++ private ValidatableResponse restore(String backupArn, String target, String overrides) {+ return ddb("RestoreTableFromBackup", """+ {"BackupArn": "%s", "TargetTableName": "%s", %s}+ """.formatted(backupArn, target, overrides));+ }++ private static BigDecimal decimalAt(ValidatableResponse response, String pointer) throws Exception {+ return EXACT.readTree(response.extract().asString()).at(pointer).decimalValue();+ }++ private ValidatableResponse ddb(String action, String body) {+ return given()+ .header("X-Amz-Target", "DynamoDB_20120810." + action)+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body(body)+ .when().post("/")+ .then();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex 5e19bcd51d..785435c017 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.dynamodb;+import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;@@ -8,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.PersistentStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;@@ -16,16 +18,19 @@ import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.function.Executable;+import org.junit.jupiter.api.io.TempDir;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.nio.charset.StandardCharsets;+import java.nio.file.Path;import java.util.ArrayList;import java.util.Arrays;import java.util.HashSet;@@ -4587,4 +4592,34 @@ class DynamoDbServiceTest {assertEquals("InterruptedByRestart", restarted.describeImport(importDesc.getImportArn()).getFailureCode());assertEquals("FAILED", restarted.describeExport(exportDesc.getExportArn()).getExportStatus());}++ @Test+ void backupKeepsItsSnapshotAcrossLaterWritesAndARestart(@TempDir Path dir) {+ RegionResolver resolver = new RegionResolver("us-east-1", "000000000000");+ StorageBackend<String, TableDefinition> tables = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> items = new InMemoryStorage<>();+ Path backupFile = dir.resolve("dynamodb-backups.json");+ TypeReference<Map<String, TableBackup>> backupType = new TypeReference<>() {};+ DynamoDbService before = new DynamoDbService(tables, items, null, null,+ new PersistentStorage<>(backupFile, backupType), resolver, null, null, null, mapper, 0, 0);+ before.createTable("Users", List.of(new KeySchemaElement("userId", "HASH")),+ List.of(new AttributeDefinition("userId", "S")), 5L, 5L, "us-east-1");+ before.putItem("Users", item("userId", "u1"), null, null, null, "us-east-1", null);+ TableBackup backup = before.createBackup("Users", "nightly", "us-east-1");+ before.putItem("Users", item("userId", "u2"), null, null, null, "us-east-1", null);+ before.updateTimeToLive("Users", "expiresAt", true, "us-east-1");+ assertFalse(before.describeBackup(backup.getBackupArn(), "us-east-1").getTable().isTtlEnabled());++ PersistentStorage<String, TableBackup> reloadedBackups = new PersistentStorage<>(backupFile, backupType);+ reloadedBackups.load();+ DynamoDbService restarted = new DynamoDbService(tables, items, null, null, reloadedBackups, resolver,+ null, null, null, mapper, 0, 0);+ TableBackup reloaded = restarted.describeBackup(backup.getBackupArn(), "us-east-1");++ assertEquals("nightly", reloaded.getBackupName());+ assertEquals(backup.getCreationDateTime(), reloaded.getCreationDateTime());+ assertEquals(List.of("u1"), reloaded.getItems().stream().map(i -> i.path("userId").path("S").asText()).toList());+ assertEquals(1, reloaded.getTable().getItemCount());+ assertFalse(reloaded.getTable().isTtlEnabled());+ }}