Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): add on-demand backups (#6052)

TATSUNO “Taz” Yasuhiro9b59b8997eea
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.java
new file mode 100644
index 0000000000..a2dd4e9560
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbBackupTest.java
@@ -0,0 +1,120 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.*;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.*;
+
+@DisplayName("DynamoDB On-Demand Backups")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbBackupTest {
+
+ private static DynamoDbClient ddb;
+ private static final String TABLE_NAME = "sdk-backup-test-table";
+ private static final String RESTORED_TABLE_NAME = "sdk-backup-test-restored";
+ private static String backupArn;
+
+ @BeforeAll
+ static void setup() {
+ ddb = TestFixtures.dynamoDbClient();
+ ddb.createTable(CreateTableRequest.builder()
+ .tableName(TABLE_NAME)
+ .attributeDefinitions(AttributeDefinition.builder()
+ .attributeName("pk").attributeType(ScalarAttributeType.S).build())
+ .keySchema(KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build())
+ .billingMode(BillingMode.PAY_PER_REQUEST)
+ .build());
+ ddb.putItem(PutItemRequest.builder()
+ .tableName(TABLE_NAME)
+ .item(Map.of("pk", AttributeValue.fromS("backed-up")))
+ .build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ for (String table : new String[] {TABLE_NAME, RESTORED_TABLE_NAME}) {
+ try {
+ ddb.deleteTable(DeleteTableRequest.builder().tableName(table).build());
+ } catch (ResourceNotFoundException ignored) {
+ // The restore test did not run.
+ }
+ }
+ if (backupArn != null) {
+ try {
+ ddb.deleteBackup(DeleteBackupRequest.builder().backupArn(backupArn).build());
+ } catch (BackupNotFoundException ignored) {
+ // The delete test removed it before it failed.
+ }
+ }
+ if (ddb != null) {
+ ddb.close();
+ }
+ }
+
+ @Test
+ @Order(1)
+ void createBackup_returnsCreating() {
+ BackupDetails details = ddb.createBackup(CreateBackupRequest.builder()
+ .tableName(TABLE_NAME).backupName("sdk_backup").build()).backupDetails();
+
+ backupArn = details.backupArn();
+ assertThat(backupArn).contains(":table/" + TABLE_NAME + "/backup/");
+ assertThat(details.backupStatus()).isEqualTo(BackupStatus.CREATING);
+ assertThat(details.backupType()).isEqualTo(BackupType.USER);
+ assertThat(details.backupCreationDateTime()).isNotNull();
+ }
+
+ @Test
+ @Order(2)
+ void describeBackup_returnsAvailableWithSourceTable() {
+ BackupDescription description = ddb.describeBackup(DescribeBackupRequest.builder()
+ .backupArn(backupArn).build()).backupDescription();
+
+ assertThat(description.backupDetails().backupStatus()).isEqualTo(BackupStatus.AVAILABLE);
+ assertThat(description.sourceTableDetails().tableName()).isEqualTo(TABLE_NAME);
+ assertThat(description.sourceTableDetails().itemCount()).isEqualTo(1L);
+ assertThat(description.sourceTableDetails().billingMode()).isEqualTo(BillingMode.PAY_PER_REQUEST);
+ }
+
+ @Test
+ @Order(3)
+ void listBackups_containsTheBackup() {
+ ListBackupsResponse response = ddb.listBackups(ListBackupsRequest.builder().tableName(TABLE_NAME).build());
+
+ assertThat(response.backupSummaries()).extracting(BackupSummary::backupArn).containsExactly(backupArn);
+ assertThat(response.lastEvaluatedBackupArn()).isNull();
+ }
+
+ @Test
+ @Order(4)
+ void restoreTableFromBackup_restoresTheItems() {
+ TableDescription description = ddb.restoreTableFromBackup(RestoreTableFromBackupRequest.builder()
+ .backupArn(backupArn).targetTableName(RESTORED_TABLE_NAME).build()).tableDescription();
+
+ assertThat(description.tableName()).isEqualTo(RESTORED_TABLE_NAME);
+ assertThat(description.restoreSummary().sourceBackupArn()).isEqualTo(backupArn);
+ assertThat(description.restoreSummary().restoreInProgress()).isTrue();
+
+ ddb.waiter().waitUntilTableExists(DescribeTableRequest.builder().tableName(RESTORED_TABLE_NAME).build());
+ GetItemResponse item = ddb.getItem(GetItemRequest.builder()
+ .tableName(RESTORED_TABLE_NAME)
+ .key(Map.of("pk", AttributeValue.fromS("backed-up")))
+ .build());
+ assertThat(item.item()).containsKey("pk");
+ }
+
+ @Test
+ @Order(5)
+ void deleteBackup_thenDescribeFails() {
+ BackupDescription deleted = ddb.deleteBackup(DeleteBackupRequest.builder()
+ .backupArn(backupArn).build()).backupDescription();
+ assertThat(deleted.backupDetails().backupStatus()).isEqualTo(BackupStatus.DELETED);
+
+ assertThatThrownBy(() -> ddb.describeBackup(DescribeBackupRequest.builder().backupArn(backupArn).build()))
+ .isInstanceOf(BackupNotFoundException.class);
+ backupArn = null;
+ }
+}
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index a92e5b97dd..f36e15365d 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -30,6 +30,11 @@
| `ListTagsOfResource` | List tags |
| `DescribeContinuousBackups` | Get PITR backup configuration |
| `UpdateContinuousBackups` | Enable/disable PITR |
+| `CreateBackup` | Take an on-demand backup of a table |
+| `DescribeBackup` | Get a backup and the table it was taken from |
+| `ListBackups` | List backups by table, type and creation time |
+| `DeleteBackup` | Delete a backup |
+| `RestoreTableFromBackup` | Create a new table from a backup |
| `DescribeKinesisStreamingDestination` | List Kinesis streaming destinations |
| `EnableKinesisStreamingDestination` | Enable Kinesis streaming for a table |
| `DisableKinesisStreamingDestination` | Disable Kinesis streaming for a table |
@@ -447,6 +452,22 @@ The import reads every object under the key prefix. Point it at the `data/` pref
Deviations from AWS: only `InputFormat` `DYNAMODB_JSON` with `InputCompressionType` `NONE` or `GZIP` is accepted. `CSV`, `ION` and `ZSTD` are rejected with a `ValidationException`.
+## On-demand backups
+
+`CreateBackup` copies the items and settings of a table at the time of the call. The answer says `CREATING`, and the backup is `AVAILABLE` at once. A backup outlives its table. `ListBackups` returns backups in creation order and sets `LastEvaluatedBackupArn` only when more backups remain.
+
+```bash
+BACKUP_ARN=$(aws dynamodb create-backup --table-name Users --backup-name nightly \
+ --query BackupDetails.BackupArn --output text --endpoint-url $AWS_ENDPOINT_URL)
+
+aws dynamodb restore-table-from-backup --backup-arn $BACKUP_ARN \
+ --target-table-name UsersRestored --endpoint-url $AWS_ENDPOINT_URL
+```
+
+`RestoreTableFromBackup` creates a new table with the keys, indexes, billing mode, throughput and encryption of the backup. As on AWS, the new table has no stream, TTL, point-in-time recovery, tags or deletion protection, and its table class is `STANDARD`. The overrides follow AWS. A GSI or LSI override must match a source index by key schema and projection, and a vector index override must match one by name. `BillingModeOverride` `PROVISIONED` needs throughput for the table and for every GSI that is kept.
+
+Deviations from AWS: a restore is done before the response is sent. The response still says `CREATING` with `RestoreInProgress: true`, as on AWS, but the table is `ACTIVE` right away. So `BackupInUseException` and `TableInUseException` for a restore in progress never happen. `BackupSizeBytes` is always 0. Backups made by AWS Backup (`BackupType` `AWS_BACKUP`) are not emulated.
+
## Kinesis change data capture (CDC)
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.java
new file mode 100644
index 0000000000..3770ffe070
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupArns.java
@@ -0,0 +1,60 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.AwsRegions;
+
+import java.time.Instant;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Backup ARNs: {@code <table ARN>/backup/<14-digit epoch millis>-<8 hex digits>}.
+ */
+final class DynamoDbBackupArns {
+
+ private static final Pattern RESOURCE = Pattern.compile("table/([a-zA-Z0-9_.\\-]+)/backup/([^/]{3,})");
+ private static final Pattern BACKUP_ID = Pattern.compile("\\d{14}-[0-9a-fA-F]{8}");
+
+ private DynamoDbBackupArns() {}
+
+ record BackupArn(String arn, String region, String accountId, String tableName, long creationMillis) {}
+
+ /**
+ * AWS rejects an ARN it cannot split into a table and a backup id with "Invalid Backup ARN", and
+ * one that splits but names another service, an unknown region or a malformed id with
+ * "Invalid Request: BackupArn is not valid".
+ */
+ static BackupArn parse(String input) {
+ if (input == null) {
+ throw invalidBackupArn();
+ }
+ AwsArnUtils.Arn arn;
+ try {
+ arn = AwsArnUtils.parse(input);
+ } catch (IllegalArgumentException e) {
+ throw invalidBackupArn();
+ }
+ Matcher resource = RESOURCE.matcher(arn.resource());
+ if (!resource.matches()) {
+ throw invalidBackupArn();
+ }
+ String backupId = resource.group(2);
+ if (!"dynamodb".equals(arn.service()) || !AwsRegions.isRegionId(arn.region())
+ || !BACKUP_ID.matcher(backupId).matches()) {
+ throw new AwsException("ValidationException", "Invalid Request: BackupArn is not valid", 400);
+ }
+ long creationMillis = Long.parseLong(backupId.substring(0, 14));
+ return new BackupArn(input, arn.region(), arn.accountId(), resource.group(1), creationMillis);
+ }
+
+ static String mint(String tableArn, Instant creationDateTime) {
+ return tableArn + "/backup/" + String.format("%014d", creationDateTime.toEpochMilli())
+ + "-" + UUID.randomUUID().toString().substring(0, 8);
+ }
+
+ private static AwsException invalidBackupArn() {
+ return new AwsException("ValidationException", "Invalid Backup ARN", 400);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 78c56ff7b2..cc94d92d32 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
@@ -16,6 +17,7 @@ import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
@@ -28,6 +30,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import io.github.hectorvent.floci.services.s3.S3Service;
@@ -46,6 +49,7 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -98,6 +102,7 @@ public class DynamoDbService {
private final StorageBackend<String, Map<String, JsonNode>> itemStore;
private final StorageBackend<String, ExportDescription> exportStore;
private final StorageBackend<String, ImportTableDescription> importStore;
+ private final StorageBackend<String, TableBackup> backupStore;
// Items stored per table: storageKey -> Map<itemKey, item>
// itemKey is "pk" or "pk#sk" depending on table schema
private final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();
@@ -163,6 +168,8 @@ public class DynamoDbService {
new TypeReference<Map<String, ExportDescription>>() {}),
storageFactory.create("dynamodb", "dynamodb-imports.json",
new TypeReference<Map<String, ImportTableDescription>>() {}),
+ storageFactory.create("dynamodb", "dynamodb-backups.json",
+ new TypeReference<Map<String, TableBackup>>() {}),
regionResolver, streamService, kinesisForwarder, s3Service, objectMapper,
config.services().dynamodb().vectorIndexAllocationSeconds(),
config.services().dynamodb().vectorIndexBackfillSeconds());
@@ -215,10 +222,28 @@ public class DynamoDbService {
ObjectMapper objectMapper,
int vectorIndexAllocationSeconds,
int vectorIndexBackfillSeconds) {
+ this(tableStore, itemStore, exportStore, importStore, new InMemoryStorage<>(), regionResolver,
+ streamService, kinesisForwarder, s3Service, objectMapper,
+ vectorIndexAllocationSeconds, vectorIndexBackfillSeconds);
+ }
+
+ DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
+ StorageBackend<String, Map<String, JsonNode>> itemStore,
+ StorageBackend<String, ExportDescription> exportStore,
+ StorageBackend<String, ImportTableDescription> importStore,
+ StorageBackend<String, TableBackup> backupStore,
+ RegionResolver regionResolver,
+ DynamoDbStreamService streamService,
+ KinesisStreamingForwarder kinesisForwarder,
+ S3Service s3Service,
+ ObjectMapper objectMapper,
+ int vectorIndexAllocationSeconds,
+ int vectorIndexBackfillSeconds) {
this.tableStore = tableStore;
this.itemStore = itemStore;
this.exportStore = exportStore;
this.importStore = importStore;
+ this.backupStore = backupStore;
this.regionResolver = regionResolver;
this.streamService = streamService;
this.kinesisForwarder = kinesisForwarder;
@@ -4781,6 +4806,124 @@ public class DynamoDbService {
return System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");
}
+ // --- Backup Operations ---
+
+ // The live table object is shared and later mutated in place, so a backup keeps its own copy.
+ private static final ObjectMapper SNAPSHOT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
+
+ private static final Comparator<TableBackup> BACKUP_ORDER = Comparator
+ .comparing(TableBackup::getCreationDateTime)
+ .thenComparing(TableBackup::getBackupArn);
+
+ public record ListBackupsResult(List<TableBackup> backups, String lastEvaluatedBackupArn) {}
+
+ public TableBackup createBackup(String tableName, String backupName, String region) {
+ String canonicalTableName = canonicalTableName(region, tableName);
+ String storageKey = regionKey(region, canonicalTableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .filter(t -> !"CREATING".equals(t.getTableStatus()))
+ .orElseThrow(() -> new AwsException("TableNotFoundException",
+ "Table not found: " + canonicalTableName, 400));
+ ConcurrentSkipListMap<String, JsonNode> items = itemsByTable.get(scopedItemsKey(storageKey));
+ List<JsonNode> snapshot = new ArrayList<>();
+ if (items != null) {
+ items.values().forEach(item -> snapshot.add(item.deepCopy()));
+ }
+ TableDefinition source = snapshotOf(table);
+ source.setItemCount(snapshot.size());
+
+ Instant now = Instant.now().truncatedTo(ChronoUnit.MILLIS);
+ TableBackup backup = new TableBackup();
+ backup.setBackupArn(DynamoDbBackupArns.mint(table.getTableArn(), now));
+ backup.setBackupName(backupName);
+ backup.setBackupType("USER");
+ backup.setCreationDateTime(now);
+ backup.setSizeBytes(source.getTableSizeBytes());
+ backup.setRegion(region);
+ backup.setTable(source);
+ backup.setItems(snapshot);
+ backupStore.put(backup.getBackupArn(), backup);
+ LOG.infov("Created backup {0} of table {1}", backup.getBackupArn(), canonicalTableName);
+ return backup;
+ }
+
+ public TableBackup describeBackup(String backupArn, String region) {
+ DynamoDbBackupArns.BackupArn arn = DynamoDbBackupArns.parse(backupArn);
+ if (!arn.accountId().equals(regionResolver.getAccountId())) {
+ throw new AwsException("AccessDeniedException", "Access is denied", 400);
+ }
+ return backupStore.get(backupArn)
+ .filter(backup -> backup.getRegion().equals(region))
+ .orElseThrow(() -> new AwsException("BackupNotFoundException",
+ "Backup not found: " + backupArn, 400));
+ }
+
+ public TableBackup deleteBackup(String backupArn, String region) {
+ TableBackup backup = describeBackup(backupArn, region);
+ backupStore.delete(backupArn);
+ LOG.infov("Deleted backup {0}", backupArn);
+ return backup;
+ }
+
+ /**
+ * Backups in creation order. AWS resumes after the position an ExclusiveStartBackupArn names,
+ * even when no such backup exists, so the position comes from the ARN itself.
+ */
+ public ListBackupsResult listBackups(String tableName, Integer limit, Instant lowerBound, Instant upperBound,
+ String exclusiveStartBackupArn, String backupType, String region) {
+ DynamoDbBackupArns.BackupArn start = exclusiveStartBackupArn != null
+ ? DynamoDbBackupArns.parse(exclusiveStartBackupArn) : null;
+ Set<String> types = "ALL".equals(backupType) ? Set.of("USER", "SYSTEM") : Set.of(backupType);
+ List<TableBackup> matching = backupStore.scan(key -> true).stream()
+ .filter(backup -> backup.getRegion().equals(region))
+ .filter(backup -> tableName == null || tableName.equals(backup.getTable().getTableName()))
+ .filter(backup -> types.contains(backup.getBackupType()))
+ .filter(backup -> lowerBound == null || !backup.getCreationDateTime().isBefore(lowerBound))
+ .filter(backup -> upperBound == null || backup.getCreationDateTime().isBefore(upperBound))
+ .filter(backup -> start == null || isAfter(backup, start))
+ .sorted(BACKUP_ORDER)
+ .toList();
+ if (limit == null || matching.size() <= limit) {
+ return new ListBackupsResult(matching, null);
+ }
+ List<TableBackup> page = matching.subList(0, limit);
+ return new ListBackupsResult(page, page.getLast().getBackupArn());
+ }
+
+ static TableDefinition snapshotOf(TableDefinition table) {
+ return SNAPSHOT_MAPPER.convertValue(table, TableDefinition.class);
+ }
+
+ void requireRestoreTarget(String tableName, String region) {
+ tableStore.get(regionKey(region, tableName)).ifPresent(table -> {
+ if ("CREATING".equals(table.getTableStatus())) {
+ throw new AwsException("TableInUseException",
+ "Table is in the process of being created: " + tableName, 400);
+ }
+ throw new AwsException("TableAlreadyExistsException", "Table already exists: " + tableName, 400);
+ });
+ }
+
+ /** Loads the restored items into the table a restore created and makes it ACTIVE. */
+ public void finishRestore(String tableName, List<JsonNode> items, String region) {
+ String storageKey = regionKey(region, tableName);
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(tableName));
+ ConcurrentSkipListMap<String, JsonNode> restored = new ConcurrentSkipListMap<>();
+ for (JsonNode item : items) {
+ restored.put(buildItemKey(table, item), item.deepCopy());
+ }
+ itemsByTable.put(scopedItemsKey(storageKey), restored);
+ persistItems(storageKey);
+ activateTable(tableName, region);
+ LOG.infov("Restored {0} items into table {1}", items.size(), tableName);
+ }
+
+ private static boolean isAfter(TableBackup backup, DynamoDbBackupArns.BackupArn start) {
+ int byTime = Long.compare(backup.getCreationDateTime().toEpochMilli(), start.creationMillis());
+ return 0 < byTime || (byTime == 0 && 0 < backup.getBackupArn().compareTo(start.arn()));
+ }
+
// --- Import Operations ---
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
index 035db086c8..791dc03bbd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
@@ -10,8 +10,10 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.SmithyValidationErrors;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.CreateTableRequest;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.GsiThroughputUpdate;
+import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.IndexOverride;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.KinesisDestination;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.OnDemandThroughput;
+import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.RestoreOverrides;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.TableSettings;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.Throughput;
import io.github.hectorvent.floci.services.dynamodb.NativeDynamoDbTableService.UpdateTableRequest;
@@ -23,6 +25,7 @@ import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.SearchSchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
@@ -30,6 +33,8 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.core.Response;
+import java.math.BigDecimal;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -52,6 +57,9 @@ public class NativeDynamoDbJsonHandler {
private static final String SSE_TYPE_KMS = "KMS";
private static final Pattern INDEX_NAME_CHARS = Pattern.compile("[a-zA-Z0-9_.\\-]+");
+ // The order AWS prints the enum constraint in.
+ private static final List<String> BACKUP_TYPE_FILTERS = List.of("ALL", "AWS_BACKUP", "USER", "SYSTEM");
+ private static final int MAX_LIST_BACKUPS_LIMIT = 100;
private final DynamoDbService dynamoDbService;
private final NativeDynamoDbTableService tableService;
@@ -95,6 +103,11 @@ public class NativeDynamoDbJsonHandler {
case "UpdateTimeToLive" -> handleUpdateTimeToLive(request, region);
case "DescribeContinuousBackups" -> handleDescribeContinuousBackups(request, region);
case "UpdateContinuousBackups" -> handleUpdateContinuousBackups(request, region);
+ case "CreateBackup" -> handleCreateBackup(request, region);
+ case "DescribeBackup" -> handleDescribeBackup(request, region);
+ case "ListBackups" -> handleListBackups(request, region);
+ case "DeleteBackup" -> handleDeleteBackup(request, region);
+ case "RestoreTableFromBackup" -> handleRestoreTableFromBackup(request, region);
case "TransactWriteItems" -> handleTransactWriteItems(request, region);
case "TransactGetItems" -> handleTransactGetItems(request, region);
case "TagResource" -> handleTagResource(request, region);
@@ -1879,6 +1892,352 @@ public class NativeDynamoDbJsonHandler {
return Response.ok(response).build();
}
+ private Response handleCreateBackup(JsonNode request, String region) {
+ String tableName = requireTableNameParameter(request, "TableName");
+ DynamoDbTableNames.resolveWithRegion(tableName, region);
+ JsonNode backupNameNode = request.path("BackupName");
+ if (backupNameNode.isMissingNode() || backupNameNode.isNull()) {
+ throwValidationErrors(List.of(
+ "Value null at 'backupName' failed to satisfy constraint: Member must not be null"));
+ }
+ String backupName = backupNameNode.asText();
+ throwValidationErrors(nameViolations(backupName, "backupName"));
+
+ TableBackup backup = dynamoDbService.createBackup(tableName, backupName, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("BackupDetails", backupDetailsNode(backup, "CREATING"));
+ return Response.ok(response).build();
+ }
+
+ private Response handleDescribeBackup(JsonNode request, String region) {
+ TableBackup backup = dynamoDbService.describeBackup(textOrNull(request, "BackupArn"), region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("BackupDescription", backupDescriptionNode(backup, "AVAILABLE"));
+ return Response.ok(response).build();
+ }
+
+ private Response handleDeleteBackup(JsonNode request, String region) {
+ TableBackup backup = dynamoDbService.deleteBackup(textOrNull(request, "BackupArn"), region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("BackupDescription", backupDescriptionNode(backup, "DELETED"));
+ return Response.ok(response).build();
+ }
+
+ private Response handleListBackups(JsonNode request, String region) {
+ String tableName = request.hasNonNull("TableName")
+ ? DynamoDbTableNames.resolveWithRegion(request.get("TableName").asText(), region).name()
+ : null;
+ List<String> violations = new ArrayList<>();
+ Integer limit = request.hasNonNull("Limit") ? request.get("Limit").asInt() : null;
+ if (limit != null && limit < 1) {
+ violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "
+ + "Member must have value greater than or equal to 1");
+ }
+ if (limit != null && MAX_LIST_BACKUPS_LIMIT < limit) {
+ violations.add("Value '" + limit + "' at 'limit' failed to satisfy constraint: "
+ + "Member must have value less than or equal to " + MAX_LIST_BACKUPS_LIMIT);
+ }
+ String backupType = request.hasNonNull("BackupType") ? request.get("BackupType").asText() : "USER";
+ if (!BACKUP_TYPE_FILTERS.contains(backupType)) {
+ violations.add("Value '" + backupType + "' at 'backupType' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: " + BACKUP_TYPE_FILTERS);
+ }
+ throwValidationErrors(violations);
+ Instant lowerBound = timestampOrNull(request, "TimeRangeLowerBound");
+ Instant upperBound = timestampOrNull(request, "TimeRangeUpperBound");
+ if (lowerBound != null && upperBound != null && upperBound.isBefore(lowerBound)) {
+ throw new AwsException("ValidationException", "One or more parameter values were invalid: "
+ + "Time range lower bound must be less than or equal to upper bound", 400);
+ }
+
+ DynamoDbService.ListBackupsResult result = dynamoDbService.listBackups(tableName, limit, lowerBound,
+ upperBound, textOrNull(request, "ExclusiveStartBackupArn"), backupType, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode summaries = response.putArray("BackupSummaries");
+ for (TableBackup backup : result.backups()) {
+ summaries.add(backupSummaryNode(backup));
+ }
+ if (result.lastEvaluatedBackupArn() != null) {
+ response.put("LastEvaluatedBackupArn", result.lastEvaluatedBackupArn());
+ }
+ return Response.ok(response).build();
+ }
+
+ /**
+ * Restores synchronously. The answer still reports the table CREATING with the restore in
+ * progress, as AWS does, but the table is ACTIVE with every item by the time it is sent.
+ */
+ private Response handleRestoreTableFromBackup(JsonNode request, String region) {
+ String backupArn = textOrNull(request, "BackupArn");
+ DynamoDbBackupArns.parse(backupArn);
+ String targetTableName = requireTableNameParameter(request, "TargetTableName");
+ List<String> violations = new ArrayList<>(throughputViolations(request.path("ProvisionedThroughputOverride"),
+ "provisionedThroughputOverride"));
+ JsonNode billingModeOverride = request.path("BillingModeOverride");
+ if (billingModeOverride.isTextual() && !"PROVISIONED".equals(billingModeOverride.asText())
+ && !"PAY_PER_REQUEST".equals(billingModeOverride.asText())) {
+ violations.add("Value '" + billingModeOverride.asText() + "' at 'billingModeOverride' failed to "
+ + "satisfy constraint: Member must satisfy enum value set: [PROVISIONED, PAY_PER_REQUEST]");
+ }
+ violations.addAll(nameViolations(targetTableName, "targetTableName"));
+ JsonNode gsiOverrides = request.path("GlobalSecondaryIndexOverride");
+ if (gsiOverrides.isArray()) {
+ int position = 0;
+ for (JsonNode gsiOverride : gsiOverrides) {
+ position++;
+ String member = "globalSecondaryIndexOverride." + position + ".member";
+ violations.addAll(restoreIndexNameViolations(gsiOverride.path("IndexName"), member + ".indexName"));
+ violations.addAll(throughputViolations(gsiOverride.path("ProvisionedThroughput"),
+ member + ".provisionedThroughput"));
+ }
+ }
+ throwValidationErrors(violations);
+
+ TableBackup backup = dynamoDbService.describeBackup(backupArn, region);
+ TableDefinition table = tableService.restoreTable(DynamoDbService.snapshotOf(backup.getTable()),
+ targetTableName, parseRestoreOverrides(request), region);
+ ObjectNode description = tableToNode(table);
+ description.set("RestoreSummary", restoreSummaryNode(backup.getBackupArn(),
+ backup.getTable().getTableArn(), backup.getCreationDateTime()));
+ dynamoDbService.finishRestore(targetTableName, backup.getItems(), region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("TableDescription", description);
+ return Response.ok(response).build();
+ }
+
+ private static RestoreOverrides parseRestoreOverrides(JsonNode request) {
+ JsonNode throughputNode = request.path("ProvisionedThroughputOverride");
+ Throughput throughput = throughputNode.isObject()
+ ? new Throughput(throughputNode.path("ReadCapacityUnits").asLong(),
+ throughputNode.path("WriteCapacityUnits").asLong())
+ : null;
+ List<VectorIndex> vectorIndexes = null;
+ JsonNode vectorArray = request.path("VectorIndexOverride");
+ if (vectorArray.isArray()) {
+ vectorIndexes = new ArrayList<>();
+ int position = 0;
+ for (JsonNode vectorNode : vectorArray) {
+ position++;
+ vectorIndexes.add(parseVectorIndex(vectorNode, "vectorIndexOverride." + position + ".member"));
+ }
+ }
+ JsonNode sse = request.path("SSESpecificationOverride");
+ return new RestoreOverrides(textOrNull(request, "BillingModeOverride"), throughput,
+ parseOnDemandThroughput(request.path("OnDemandThroughputOverride")),
+ parseIndexOverrides(request.path("GlobalSecondaryIndexOverride"), "globalSecondaryIndexOverride"),
+ parseIndexOverrides(request.path("LocalSecondaryIndexOverride"), "localSecondaryIndexOverride"),
+ vectorIndexes,
+ sse.isObject() ? sse.path("Enabled").asBoolean(false) : null,
+ textOrNull(sse, "SSEType"), textOrNull(sse, "KMSMasterKeyId"));
+ }
+
+ private static List<IndexOverride> parseIndexOverrides(JsonNode array, String memberPrefix) {
+ if (!array.isArray()) {
+ return null;
+ }
+ List<IndexOverride> overrides = new ArrayList<>();
+ int position = 0;
+ for (JsonNode node : array) {
+ position++;
+ String indexName = requireIndexName(node, memberPrefix + "." + position + ".member");
+ List<KeySchemaElement> keySchema = new ArrayList<>();
+ node.path("KeySchema").forEach(ks -> keySchema.add(
+ new KeySchemaElement(ks.path("AttributeName").asText(), ks.path("KeyType").asText())));
+ List<String> nonKeyAttributes = new ArrayList<>();
+ node.path("Projection").path("NonKeyAttributes").forEach(a -> nonKeyAttributes.add(a.asText()));
+ JsonNode throughputNode = node.path("ProvisionedThroughput");
+ Throughput throughput = throughputNode.isObject()
+ ? new Throughput(throughputNode.path("ReadCapacityUnits").asLong(),
+ throughputNode.path("WriteCapacityUnits").asLong())
+ : null;
+ overrides.add(new IndexOverride(indexName, keySchema,
+ node.path("Projection").path("ProjectionType").asText("ALL"), nonKeyAttributes, throughput,
+ parseOnDemandThroughput(node.path("OnDemandThroughput"))));
+ }
+ return overrides;
+ }
+
+ /** AWS names the member 'TableName' in this message even when the request calls it something else. */
+ private static String requireTableNameParameter(JsonNode request, String member) {
+ JsonNode node = request.path(member);
+ if (node.isMissingNode() || node.isNull()) {
+ throw new AwsException("ValidationException",
+ "The parameter 'TableName' is required but was not present in the request", 400);
+ }
+ return node.asText();
+ }
+
+ /** Table, index and backup names share one pattern and length range. */
+ private static List<String> nameViolations(String name, String memberPath) {
+ String value = "Value '" + name + "' at '" + memberPath + "' failed to satisfy constraint: ";
+ List<String> violations = new ArrayList<>();
+ if (!INDEX_NAME_CHARS.matcher(name).matches()) {
+ violations.add(value + "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+");
+ }
+ if (name.length() < 3) {
+ violations.add(value + "Member must have length greater than or equal to 3");
+ }
+ if (255 < name.length()) {
+ violations.add(value + "Member must have length less than or equal to 255");
+ }
+ return violations;
+ }
+
+ /** AWS lists WriteCapacityUnits before ReadCapacityUnits. */
+ private static List<String> throughputViolations(JsonNode throughput, String memberPath) {
+ List<String> violations = new ArrayList<>();
+ if (!throughput.isObject()) {
+ return violations;
+ }
+ for (String member : List.of("WriteCapacityUnits", "ReadCapacityUnits")) {
+ JsonNode value = throughput.path(member);
+ String constraint = " at '" + memberPath + "." + Character.toLowerCase(member.charAt(0))
+ + member.substring(1) + "' failed to satisfy constraint: ";
+ if (value.isMissingNode() || value.isNull()) {
+ violations.add("Value null" + constraint + "Member must not be null");
+ } else if (value.asLong() < 1) {
+ violations.add("Value '" + value.asText() + "'" + constraint
+ + "Member must have value greater than or equal to 1");
+ }
+ }
+ return violations;
+ }
+
+ /** Unlike CreateTable, a restore lists the length violation of an index name before the pattern. */
+ private static List<String> restoreIndexNameViolations(JsonNode indexName, String memberPath) {
+ String constraint = " at '" + memberPath + "' failed to satisfy constraint: ";
+ if (indexName.isMissingNode() || indexName.isNull()) {
+ return List.of("Value null" + constraint + "Member must not be null");
+ }
+ String value = "Value '" + indexName.asText() + "'" + constraint;
+ List<String> violations = new ArrayList<>();
+ if (indexName.asText().length() < 3) {
+ violations.add(value + "Member must have length greater than or equal to 3");
+ }
+ if (255 < indexName.asText().length()) {
+ violations.add(value + "Member must have length less than or equal to 255");
+ }
+ if (!INDEX_NAME_CHARS.matcher(indexName.asText()).matches()) {
+ violations.add(value + "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+");
+ }
+ return violations;
+ }
+
+ private static String textOrNull(JsonNode node, String member) {
+ return node.hasNonNull(member) ? node.get(member).asText() : null;
+ }
+
+ private static Instant timestampOrNull(JsonNode request, String member) {
+ JsonNode node = request.path(member);
+ return node.isNumber() ? Instant.ofEpochMilli(node.decimalValue().movePointRight(3).longValue()) : null;
+ }
+
+ private static BigDecimal epochSeconds(Instant instant) {
+ return BigDecimal.valueOf(instant.toEpochMilli(), 3);
+ }
+
+ private ObjectNode backupDetailsNode(TableBackup backup, String status) {
+ ObjectNode node = objectMapper.createObjectNode();
+ node.put("BackupArn", backup.getBackupArn());
+ node.put("BackupName", backup.getBackupName());
+ node.put("BackupSizeBytes", backup.getSizeBytes());
+ node.put("BackupStatus", status);
+ node.put("BackupType", backup.getBackupType());
+ node.put("BackupCreationDateTime", epochSeconds(backup.getCreationDateTime()));
+ if (backup.getExpiryDateTime() != null) {
+ node.put("BackupExpiryDateTime", epochSeconds(backup.getExpiryDateTime()));
+ }
+ return node;
+ }
+
+ private ObjectNode backupSummaryNode(TableBackup backup) {
+ ObjectNode node = backupDetailsNode(backup, "AVAILABLE");
+ TableDefinition table = backup.getTable();
+ node.put("TableName", table.getTableName());
+ node.put("TableId", table.getTableId());
+ node.put("TableArn", table.getTableArn());
+ return node;
+ }
+
+ /** The source table parts are DescribeTable's, minus the members that only describe a live table. */
+ private ObjectNode backupDescriptionNode(TableBackup backup, String status) {
+ TableDefinition table = backup.getTable();
+ ObjectNode described = tableToNode(table);
+
+ ObjectNode source = objectMapper.createObjectNode();
+ source.put("TableName", table.getTableName());
+ source.put("TableId", table.getTableId());
+ source.put("TableArn", table.getTableArn());
+ source.put("TableSizeBytes", table.getTableSizeBytes());
+ source.set("KeySchema", described.get("KeySchema"));
+ source.set("TableCreationDateTime", described.get("CreationDateTime"));
+ ObjectNode throughput = (ObjectNode) described.get("ProvisionedThroughput");
+ throughput.remove("NumberOfDecreasesToday");
+ source.set("ProvisionedThroughput", throughput);
+ if (described.has("OnDemandThroughput")) {
+ source.set("OnDemandThroughput", described.get("OnDemandThroughput"));
+ }
+ source.put("ItemCount", table.getItemCount());
+ source.put("BillingMode", "PAY_PER_REQUEST".equals(table.getBillingMode()) ? "PAY_PER_REQUEST" : "PROVISIONED");
+
+ ObjectNode features = objectMapper.createObjectNode();
+ if (described.has("LocalSecondaryIndexes")) {
+ for (JsonNode lsi : described.get("LocalSecondaryIndexes")) {
+ ((ObjectNode) lsi).remove(List.of("IndexArn", "IndexSizeBytes", "ItemCount"));
+ }
+ features.set("LocalSecondaryIndexes", described.get("LocalSecondaryIndexes"));
+ }
+ if (described.has("GlobalSecondaryIndexes")) {
+ for (JsonNode gsi : described.get("GlobalSecondaryIndexes")) {
+ ((ObjectNode) gsi).remove(List.of("IndexArn", "IndexStatus", "IndexSizeBytes", "ItemCount"));
+ ((ObjectNode) gsi.get("ProvisionedThroughput")).remove("NumberOfDecreasesToday");
+ }
+ features.set("GlobalSecondaryIndexes", described.get("GlobalSecondaryIndexes"));
+ }
+ if (table.isStreamEnabled()) {
+ ObjectNode stream = features.putObject("StreamDescription");
+ stream.put("StreamEnabled", true);
+ stream.put("StreamViewType", table.getStreamViewType());
+ }
+ if (table.isTtlEnabled()) {
+ ObjectNode ttl = features.putObject("TimeToLiveDescription");
+ ttl.put("AttributeName", table.getTtlAttributeName());
+ ttl.put("TimeToLiveStatus", "ENABLED");
+ }
+ if (described.has("SSEDescription")) {
+ features.set("SSEDescription", described.get("SSEDescription"));
+ }
+ if (described.has("VectorIndexes")) {
+ for (JsonNode vectorIndex : described.get("VectorIndexes")) {
+ ((ObjectNode) vectorIndex).remove(
+ List.of("IndexArn", "IndexStatus", "Backfilling", "IndexSizeBytes", "ItemCount"));
+ }
+ features.set("VectorIndexes", described.get("VectorIndexes"));
+ }
+
+ ObjectNode node = objectMapper.createObjectNode();
+ node.set("BackupDetails", backupDetailsNode(backup, status));
+ node.set("SourceTableDetails", source);
+ node.set("SourceTableFeatureDetails", features);
+ return node;
+ }
+
+ private ObjectNode restoreSummaryNode(String sourceBackupArn, String sourceTableArn, Instant restoreDateTime) {
+ ObjectNode node = objectMapper.createObjectNode();
+ if (sourceBackupArn != null) {
+ node.put("SourceBackupArn", sourceBackupArn);
+ }
+ node.put("SourceTableArn", sourceTableArn);
+ node.put("RestoreDateTime", epochSeconds(restoreDateTime));
+ node.put("RestoreInProgress", true);
+ return node;
+ }
+
private Response handleTransactWriteItems(JsonNode request, String region) {
JsonNode transactItemsNode = requireValidTransactRequest(request);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java
index fb287ece19..a9ab0402ab 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbTableService.java
@@ -10,18 +10,21 @@ import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestin
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.VectorIndex;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
/**
- * Table control operations on the native DynamoDB engine: CreateTable, UpdateTable and the Kinesis
- * streaming destination toggles.
+ * Table control operations on the native DynamoDB engine: CreateTable, UpdateTable, the table a
+ * restore creates and the Kinesis streaming destination toggles.
*
* <p>Owns the sequence each operation runs against the backend: the validation that must fail
* before any side effect, the underlying create or update, the settings applied to the stored
@@ -74,6 +77,18 @@ public class NativeDynamoDbTableService {
record KinesisDestination(String tableName, String streamArn, String destinationStatus,
String approximateCreationDateTimePrecision) {}
+ /** One GlobalSecondaryIndexOverride or LocalSecondaryIndexOverride entry; LSIs carry no throughput. */
+ record IndexOverride(String indexName, List<KeySchemaElement> keySchema, String projectionType,
+ List<String> nonKeyAttributes, Throughput provisionedThroughput,
+ OnDemandThroughput onDemandThroughput) {}
+
+ /** The overrides of a restore request. A null member or list means the request did not override it. */
+ record RestoreOverrides(String billingMode, Throughput provisionedThroughput,
+ OnDemandThroughput onDemandThroughput,
+ List<IndexOverride> globalSecondaryIndexes, List<IndexOverride> localSecondaryIndexes,
+ List<VectorIndex> vectorIndexes,
+ Boolean sseEnabled, String sseType, String kmsMasterKeyId) {}
+
private final DynamoDbService dynamoDbService;
private final DynamoDbStreamService streamService;
private final KinesisService kinesisService;
@@ -165,6 +180,225 @@ public class NativeDynamoDbTableService {
return table;
}
+ /**
+ * Creates the target of a restore in CREATING from {@code source}, a copy of the table the
+ * backup holds, with the request's overrides applied. Like AWS, the new table gets the keys,
+ * indexes, capacity and encryption, and none of the stream, TTL, point-in-time recovery, tags,
+ * deletion protection or table class.
+ */
+ TableDefinition restoreTable(TableDefinition source, String targetTableName, RestoreOverrides overrides,
+ String region) {
+ String billingMode = overrides.billingMode() != null
+ ? overrides.billingMode()
+ : "PAY_PER_REQUEST".equals(source.getBillingMode()) ? "PAY_PER_REQUEST" : "PROVISIONED";
+ boolean provisioned = "PROVISIONED".equals(billingMode);
+ checkThroughputOverrides(overrides.billingMode(), provisioned, overrides.provisionedThroughput(),
+ overrides.onDemandThroughput(), "");
+ Throughput throughput = null;
+ if (provisioned) {
+ throughput = overrides.provisionedThroughput() != null
+ ? overrides.provisionedThroughput()
+ : new Throughput(source.getProvisionedThroughput().getReadCapacityUnits(),
+ source.getProvisionedThroughput().getWriteCapacityUnits());
+ }
+
+ List<GlobalSecondaryIndex> gsis = restoredGlobalSecondaryIndexes(source, overrides, provisioned);
+ List<LocalSecondaryIndex> lsis = restoredLocalSecondaryIndexes(source, overrides);
+ List<DynamoDbService.VectorIndexCreate> vectorIndexes = restoredVectorIndexes(source, overrides);
+
+ Set<String> usedAttributes = new HashSet<>();
+ source.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName()));
+ gsis.forEach(g -> g.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName())));
+ lsis.forEach(l -> l.getKeySchema().forEach(k -> usedAttributes.add(k.getAttributeName())));
+ vectorIndexes.forEach(v -> usedAttributes.addAll(v.index().getSearchSchemaAttributeNames()));
+ List<AttributeDefinition> attributeDefinitions = source.getAttributeDefinitions().stream()
+ .filter(a -> usedAttributes.contains(a.getAttributeName()))
+ .toList();
+
+ Boolean sseEnabled = overrides.sseEnabled() != null ? overrides.sseEnabled() : source.isSseEnabled();
+ String sseType = overrides.sseEnabled() != null ? overrides.sseType() : source.getSseType();
+ String kmsMasterKeyId = overrides.sseEnabled() != null ? overrides.kmsMasterKeyId() : source.getKmsMasterKeyArn();
+ OnDemandThroughput onDemand = provisioned ? null : mergedOnDemand(overrides.onDemandThroughput(),
+ source.getOnDemandMaxReadRequestUnits(), source.getOnDemandMaxWriteRequestUnits());
+ TableSettings settings = new TableSettings(billingMode, false, null, onDemand, null, null,
+ sseEnabled, sseType, kmsMasterKeyId);
+ dynamoDbService.requireRestoreTarget(targetTableName, region);
+ return createTable(new CreateTableRequest(targetTableName, new ArrayList<>(source.getKeySchema()),
+ new ArrayList<>(attributeDefinitions), throughput, gsis, lsis, vectorIndexes, Map.of(), settings),
+ "CREATING", region);
+ }
+
+ private static List<GlobalSecondaryIndex> restoredGlobalSecondaryIndexes(TableDefinition source,
+ RestoreOverrides overrides, boolean provisioned) {
+ List<GlobalSecondaryIndex> restored = new ArrayList<>();
+ if (overrides.globalSecondaryIndexes() == null) {
+ for (GlobalSecondaryIndex gsi : source.getGlobalSecondaryIndexes()) {
+ if ("PROVISIONED".equals(overrides.billingMode())) {
+ throw invalidRestore("Must override ProvisionedThroughput if BillingMode is overridden to "
+ + "PROVISIONED for index " + gsi.getIndexName());
+ }
+ restored.add(restoredGsi(gsi.getIndexName(), gsi, provisioned, null, null));
+ }
+ return restored;
+ }
+ for (IndexOverride override : overrides.globalSecondaryIndexes()) {
+ GlobalSecondaryIndex match = source.getGlobalSecondaryIndexes().stream()
+ .filter(gsi -> sameIndex(override, gsi.getKeySchema(), gsi.getProjectionType(),
+ gsi.getNonKeyAttributes()))
+ .findFirst()
+ .orElseThrow(() -> indexMismatch(override.indexName()));
+ String forIndex = " for index " + override.indexName();
+ checkThroughputOverrides(overrides.billingMode(), provisioned, override.provisionedThroughput(),
+ override.onDemandThroughput(), forIndex);
+ if (provisioned && override.provisionedThroughput() == null) {
+ throw invalidRestore("Must specify provisioned throughput" + forIndex);
+ }
+ restored.add(restoredGsi(override.indexName(), match, provisioned, override.provisionedThroughput(),
+ override.onDemandThroughput()));
+ }
+ return restored;
+ }
+
+ /**
+ * The table and each index override go through the same checks in this order. An empty
+ * OnDemandThroughput is ignored, and the message names the first member the override sets.
+ */
+ private static void checkThroughputOverrides(String billingModeOverride, boolean provisioned,
+ Throughput throughput, OnDemandThroughput onDemand, String suffix) {
+ if ("PAY_PER_REQUEST".equals(billingModeOverride) && throughput != null) {
+ throw invalidRestore("Cannot override ProvisionedThroughput if BillingMode is overridden to "
+ + "PAY_PER_REQUEST" + suffix);
+ }
+ if ("PROVISIONED".equals(billingModeOverride) && throughput == null) {
+ throw invalidRestore("Must override ProvisionedThroughput if BillingMode is overridden to PROVISIONED"
+ + suffix);
+ }
+ if (onDemand != null && (onDemand.maxReadRequestUnits() != null || onDemand.maxWriteRequestUnits() != null)) {
+ String cannot = "Cannot override "
+ + (onDemand.maxReadRequestUnits() != null ? "MaxReadRequestUnits" : "MaxWriteRequestUnits")
+ + " for OnDemandThroughput ";
+ if ("PROVISIONED".equals(billingModeOverride)) {
+ throw invalidRestore(cannot + "when BillingMode is PROVISIONED" + suffix);
+ }
+ if (throughput != null) {
+ throw invalidRestore(cannot + "when ProvisionedThroughputOverride is present" + suffix);
+ }
+ if (provisioned) {
+ throw invalidRestore(cannot + "unless BillingModeOverride is PAY_PER_REQUEST" + suffix);
+ }
+ }
+ if (!provisioned && throughput != null) {
+ throw invalidRestore("Cannot override ProvisionedThroughput unless BillingMode is PROVISIONED" + suffix);
+ }
+ }
+
+ /** An on-demand override replaces only the members it sets. */
+ private static OnDemandThroughput mergedOnDemand(OnDemandThroughput override, Integer maxReadRequestUnits,
+ Integer maxWriteRequestUnits) {
+ Integer maxRead = override != null && override.maxReadRequestUnits() != null
+ ? override.maxReadRequestUnits() : maxReadRequestUnits;
+ Integer maxWrite = override != null && override.maxWriteRequestUnits() != null
+ ? override.maxWriteRequestUnits() : maxWriteRequestUnits;
+ return maxRead == null && maxWrite == null ? null : new OnDemandThroughput(maxRead, maxWrite);
+ }
+
+ private static GlobalSecondaryIndex restoredGsi(String indexName, GlobalSecondaryIndex source, boolean provisioned,
+ Throughput throughput, OnDemandThroughput onDemandOverride) {
+ GlobalSecondaryIndex gsi = new GlobalSecondaryIndex(indexName, new ArrayList<>(source.getKeySchema()), null,
+ source.getProjectionType(), new ArrayList<>(source.getNonKeyAttributes()));
+ if (provisioned) {
+ gsi.getProvisionedThroughput().setReadCapacityUnits(throughput != null ? throughput.readCapacityUnits()
+ : source.getProvisionedThroughput().getReadCapacityUnits());
+ gsi.getProvisionedThroughput().setWriteCapacityUnits(throughput != null ? throughput.writeCapacityUnits()
+ : source.getProvisionedThroughput().getWriteCapacityUnits());
+ return gsi;
+ }
+ OnDemandThroughput onDemand = mergedOnDemand(onDemandOverride, source.getOnDemandMaxReadRequestUnits(),
+ source.getOnDemandMaxWriteRequestUnits());
+ if (onDemand != null) {
+ gsi.setOnDemandMaxReadRequestUnits(onDemand.maxReadRequestUnits());
+ gsi.setOnDemandMaxWriteRequestUnits(onDemand.maxWriteRequestUnits());
+ }
+ return gsi;
+ }
+
+ private static List<LocalSecondaryIndex> restoredLocalSecondaryIndexes(TableDefinition source,
+ RestoreOverrides overrides) {
+ List<LocalSecondaryIndex> restored = new ArrayList<>();
+ if (overrides.localSecondaryIndexes() == null) {
+ for (LocalSecondaryIndex lsi : source.getLocalSecondaryIndexes()) {
+ restored.add(new LocalSecondaryIndex(lsi.getIndexName(), new ArrayList<>(lsi.getKeySchema()), null,
+ lsi.getProjectionType(), new ArrayList<>(lsi.getNonKeyAttributes())));
+ }
+ return restored;
+ }
+ for (IndexOverride override : overrides.localSecondaryIndexes()) {
+ LocalSecondaryIndex match = source.getLocalSecondaryIndexes().stream()
+ .filter(lsi -> sameIndex(override, lsi.getKeySchema(), lsi.getProjectionType(),
+ lsi.getNonKeyAttributes()))
+ .findFirst()
+ .orElseThrow(() -> indexMismatch(override.indexName()));
+ restored.add(new LocalSecondaryIndex(override.indexName(), new ArrayList<>(match.getKeySchema()), null,
+ match.getProjectionType(), new ArrayList<>(match.getNonKeyAttributes())));
+ }
+ return restored;
+ }
+
+ /** Vector indexes, unlike the others, are matched by name. */
+ private static List<DynamoDbService.VectorIndexCreate> restoredVectorIndexes(TableDefinition source,
+ RestoreOverrides overrides) {
+ List<VectorIndex> kept = new ArrayList<>();
+ if (overrides.vectorIndexes() == null) {
+ kept.addAll(source.getVectorIndexes());
+ } else {
+ for (VectorIndex override : overrides.vectorIndexes()) {
+ kept.add(source.getVectorIndexes().stream()
+ .filter(v -> v.getIndexName().equals(override.getIndexName()))
+ .findFirst()
+ .orElseThrow(() -> invalidRestore("Vector index " + override.getIndexName()
+ + " does not match a vector index that existed in the source table and cannot be "
+ + "created during the restore operation")));
+ }
+ }
+ List<DynamoDbService.VectorIndexCreate> restored = new ArrayList<>();
+ for (int i = 0; i < kept.size(); i++) {
+ VectorIndex v = kept.get(i);
+ VectorIndex copy = new VectorIndex(v.getIndexName(), v.getVectorAttributeName(),
+ new ArrayList<>(v.getSearchSchema()), v.getProjectionType(),
+ new ArrayList<>(v.getNonKeyAttributes()), v.getDimensions(), v.getDistanceFunction());
+ restored.add(new DynamoDbService.VectorIndexCreate(copy, "vectorIndexOverride." + (i + 1) + ".member"));
+ }
+ return restored;
+ }
+
+ private static boolean sameIndex(IndexOverride override, List<KeySchemaElement> keySchema, String projectionType,
+ List<String> nonKeyAttributes) {
+ if (override.keySchema().size() != keySchema.size()) {
+ return false;
+ }
+ for (int i = 0; i < keySchema.size(); i++) {
+ KeySchemaElement want = override.keySchema().get(i);
+ KeySchemaElement have = keySchema.get(i);
+ if (!want.getAttributeName().equals(have.getAttributeName()) || !want.getKeyType().equals(have.getKeyType())) {
+ return false;
+ }
+ }
+ String type = projectionType != null ? projectionType : "ALL";
+ if (!type.equals(override.projectionType())) {
+ return false;
+ }
+ return !"INCLUDE".equals(type) || Set.copyOf(nonKeyAttributes).equals(Set.copyOf(override.nonKeyAttributes()));
+ }
+
+ private static AwsException indexMismatch(String indexName) {
+ return invalidRestore("Index " + indexName + " does not match a secondary index that existed in the source "
+ + "table and cannot be created during the restore operation");
+ }
+
+ private static AwsException invalidRestore(String message) {
+ return new AwsException("ValidationException", "Invalid Request: " + message, 400);
+ }
+
/**
* Updates the table, applies the request's settings and replica changes to it and persists
* the result.
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.java
new file mode 100644
index 0000000000..6ed037daf7
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/TableBackup.java
@@ -0,0 +1,59 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An on-demand backup: its details, the source table as it was when the backup was taken, and
+ * the items it holds.
+ */
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TableBackup {
+
+ private String backupArn;
+ private String backupName;
+ private String backupType;
+ private Instant creationDateTime;
+ private Instant expiryDateTime;
+ private long sizeBytes;
+ private String region;
+ private TableDefinition table;
+ private List<JsonNode> items;
+
+ public TableBackup() {
+ this.items = new ArrayList<>();
+ }
+
+ public String getBackupArn() { return backupArn; }
+ public void setBackupArn(String backupArn) { this.backupArn = backupArn; }
+
+ public String getBackupName() { return backupName; }
+ public void setBackupName(String backupName) { this.backupName = backupName; }
+
+ public String getBackupType() { return backupType; }
+ public void setBackupType(String backupType) { this.backupType = backupType; }
+
+ public Instant getCreationDateTime() { return creationDateTime; }
+ public void setCreationDateTime(Instant creationDateTime) { this.creationDateTime = creationDateTime; }
+
+ public Instant getExpiryDateTime() { return expiryDateTime; }
+ public void setExpiryDateTime(Instant expiryDateTime) { this.expiryDateTime = expiryDateTime; }
+
+ public long getSizeBytes() { return sizeBytes; }
+ public void setSizeBytes(long sizeBytes) { this.sizeBytes = sizeBytes; }
+
+ public String getRegion() { return region; }
+ public void setRegion(String region) { this.region = region; }
+
+ public TableDefinition getTable() { return table; }
+ public void setTable(TableDefinition table) { this.table = table; }
+
+ public List<JsonNode> getItems() { return items; }
+ public void setItems(List<JsonNode> items) { this.items = items != null ? items : new ArrayList<>(); }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
new file mode 100644
index 0000000000..4a8578cfd6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBackupIntegrationTest.java
@@ -0,0 +1,773 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class DynamoDbBackupIntegrationTest {
+
+ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ // JsonPath reads numbers as float, which drops whole seconds at epoch magnitude.
+ private static final ObjectMapper EXACT = new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
+ private static final String TABLE_ARN_PREFIX = "arn:aws:dynamodb:us-east-1:000000000000:table/";
+ private static final String BACKUP_ARN_PATTERN =
+ "arn:aws:dynamodb:us-east-1:000000000000:table/%s/backup/\\d{14}-[0-9a-f]{8}";
+ private static final String OTHER_REGION_AUTH =
+ "AWS4-HMAC-SHA256 Credential=AKID/20260215/eu-west-2/dynamodb/aws4_request, "
+ + "SignedHeaders=host, Signature=abc";
+ private static final String OTHER_ACCOUNT_AUTH =
+ "AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/dynamodb/aws4_request, "
+ + "SignedHeaders=host, Signature=abc";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void createBackupAnswersCreatingAndDescribeBackupAnswersAvailable() {
+ String table = "BackupIT-describe";
+ createIndexedTable(table);
+ putItem(table, "a", "1");
+ ddb("UpdateTimeToLive", """
+ {"TableName": "%s", "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "ttl"}}
+ """.formatted(table)).statusCode(200);
+
+ String backupArn = ddb("CreateBackup", """
+ {"TableName": "%s", "BackupName": "describe_me"}
+ """.formatted(table))
+ .statusCode(200)
+ .body("BackupDetails.BackupName", equalTo("describe_me"))
+ .body("BackupDetails.BackupStatus", equalTo("CREATING"))
+ .body("BackupDetails.BackupType", equalTo("USER"))
+ .body("BackupDetails.BackupSizeBytes", equalTo(0))
+ .body("BackupDetails.BackupCreationDateTime", notNullValue())
+ .body("BackupDetails", not(hasKey("BackupExpiryDateTime")))
+ .extract().path("BackupDetails.BackupArn");
+ assertTrue(backupArn.matches(BACKUP_ARN_PATTERN.formatted(table)), backupArn);
+
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")
+ .statusCode(200)
+ .body("BackupDescription.BackupDetails.BackupArn", equalTo(backupArn))
+ .body("BackupDescription.BackupDetails.BackupStatus", equalTo("AVAILABLE"))
+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table))
+ .body("BackupDescription.SourceTableDetails.TableArn", equalTo(TABLE_ARN_PREFIX + table))
+ .body("BackupDescription.SourceTableDetails.TableId", notNullValue())
+ .body("BackupDescription.SourceTableDetails.TableCreationDateTime", notNullValue())
+ .body("BackupDescription.SourceTableDetails.ItemCount", equalTo(1))
+ .body("BackupDescription.SourceTableDetails.BillingMode", equalTo("PAY_PER_REQUEST"))
+ .body("BackupDescription.SourceTableDetails.KeySchema.AttributeName", contains("pk", "sk"))
+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput.ReadCapacityUnits", equalTo(0))
+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput",
+ not(hasKey("NumberOfDecreasesToday")))
+ .body("BackupDescription.SourceTableFeatureDetails.GlobalSecondaryIndexes[0].IndexName",
+ equalTo("gsi1"))
+ .body("BackupDescription.SourceTableFeatureDetails.GlobalSecondaryIndexes[0]",
+ not(hasKey("IndexStatus")))
+ .body("BackupDescription.SourceTableFeatureDetails.LocalSecondaryIndexes[0].IndexName",
+ equalTo("lsi1"))
+ .body("BackupDescription.SourceTableFeatureDetails.LocalSecondaryIndexes[0]",
+ not(hasKey("IndexArn")))
+ .body("BackupDescription.SourceTableFeatureDetails.StreamDescription.StreamEnabled", equalTo(true))
+ .body("BackupDescription.SourceTableFeatureDetails.StreamDescription.StreamViewType",
+ equalTo("NEW_AND_OLD_IMAGES"))
+ .body("BackupDescription.SourceTableFeatureDetails.TimeToLiveDescription.AttributeName",
+ equalTo("ttl"))
+ .body("BackupDescription.SourceTableFeatureDetails.TimeToLiveDescription.TimeToLiveStatus",
+ equalTo("ENABLED"))
+ .body("BackupDescription.SourceTableFeatureDetails.SSEDescription.SSEType", equalTo("KMS"));
+ }
+
+ @Test
+ void plainTableBackupHasNoFeatureDetails() {
+ String table = "BackupIT-plain";
+ createPlainTable(table);
+ String backupArn = createBackup(table, "plain");
+
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")
+ .statusCode(200)
+ .body("BackupDescription.SourceTableDetails.BillingMode", equalTo("PROVISIONED"))
+ .body("BackupDescription.SourceTableDetails.ProvisionedThroughput.ReadCapacityUnits", equalTo(1))
+ .body("BackupDescription.SourceTableFeatureDetails.size()", equalTo(0));
+ }
+
+ @Test
+ void createBackupOfMissingTableIsTableNotFound() {
+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"abc\"}")
+ .statusCode(400)
+ .body("__type", containsString("TableNotFoundException"))
+ .body("message", equalTo("Table not found: BackupIT-missing"));
+ }
+
+ @Test
+ void createBackupValidatesItsMembers() {
+ ddb("CreateBackup", "{\"BackupName\": \"abc\"}")
+ .statusCode(400)
+ .body("message", equalTo("The parameter 'TableName' is required but was not present in the request"));
+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\"}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value null at 'backupName' failed to "
+ + "satisfy constraint: Member must not be null"));
+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"ab\"}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value 'ab' at 'backupName' failed to "
+ + "satisfy constraint: Member must have length greater than or equal to 3"));
+ ddb("CreateBackup", "{\"TableName\": \"BackupIT-missing\", \"BackupName\": \"a b\"}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value 'a b' at 'backupName' failed to "
+ + "satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));
+ ddb("CreateBackup", "{\"TableName\": \"a b\", \"BackupName\": \"c d\"}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value 'a b' at 'tableName' failed to "
+ + "satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));
+ }
+
+ @Test
+ void backupArnsAreCheckedInTwoSteps() {
+ String base = TABLE_ARN_PREFIX + "BackupIT-none/backup/";
+ for (String arn : List.of("x".repeat(40), base, base + "xy", base + "01700000000000-abcdef12/extra")) {
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"))
+ .body("message", equalTo("Invalid Backup ARN"));
+ }
+ for (String arn : List.of(base + "xyz", base + "1700000000000-abcdef12", base + "01700000000000-abcdefgh",
+ "arn:aws:s3:us-east-1:000000000000:table/BackupIT-none/backup/01700000000000-abcdef12",
+ "arn:aws:dynamodb:nowhere-1:000000000000:table/BackupIT-none/backup/01700000000000-abcdef12")) {
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: BackupArn is not valid"));
+ }
+ ddb("DeleteBackup", "{}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Backup ARN"));
+ }
+
+ @Test
+ void unknownBackupIsBackupNotFound() {
+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-ABCDEF12";
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + arn + "\"}")
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"))
+ .body("message", equalTo("Backup not found: " + arn));
+ ddb("DeleteBackup", "{\"BackupArn\": \"" + arn + "\"}")
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"));
+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + arn + "\", \"TargetTableName\": \"BackupIT-describe\"}")
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"));
+ }
+
+ @Test
+ void backupIsScopedToItsRegionAndAccount() {
+ String table = "BackupIT-scope";
+ createPlainTable(table);
+ String backupArn = createBackup(table, "scoped");
+
+ given().header("Authorization", OTHER_REGION_AUTH)
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeBackup")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"BackupArn\": \"" + backupArn + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"));
+ given().header("Authorization", OTHER_REGION_AUTH)
+ .header("X-Amz-Target", "DynamoDB_20120810.ListBackups")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"" + table + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("BackupSummaries", empty());
+ given().header("Authorization", OTHER_ACCOUNT_AUTH)
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeBackup")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"BackupArn\": \"" + backupArn + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("AccessDeniedException"))
+ .body("message", equalTo("Access is denied"));
+ }
+
+ @Test
+ void listBackupsPagesInCreationOrder() {
+ String table = "BackupIT-list";
+ createPlainTable(table);
+ String first = createBackup(table, "same_name");
+ String second = createBackup(table, "same_name");
+ String third = createBackup(table, "third");
+
+ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(first, second, third))
+ .body("BackupSummaries[0].BackupStatus", equalTo("AVAILABLE"))
+ .body("BackupSummaries[0].TableName", equalTo(table))
+ .body("BackupSummaries[0].TableArn", equalTo(TABLE_ARN_PREFIX + table))
+ .body("BackupSummaries[0].TableId", notNullValue())
+ .body("$", not(hasKey("LastEvaluatedBackupArn")));
+ ddb("ListBackups", "{\"TableName\": \"" + TABLE_ARN_PREFIX + table + "\", \"Limit\": 2}")
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(first, second))
+ .body("LastEvaluatedBackupArn", equalTo(second));
+ ddb("ListBackups", """
+ {"TableName": "%s", "Limit": 2, "ExclusiveStartBackupArn": "%s"}
+ """.formatted(table, second))
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(third))
+ .body("$", not(hasKey("LastEvaluatedBackupArn")));
+ ddb("ListBackups", """
+ {"TableName": "%s", "ExclusiveStartBackupArn": "%s"}
+ """.formatted(table, TABLE_ARN_PREFIX + table + "/backup/01700000000000-abcdef12"))
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(first, second, third));
+ }
+
+ @Test
+ void listBackupsFiltersByTableTypeAndTime() throws Exception {
+ String table = "BackupIT-filter";
+ createPlainTable(table);
+ String backupArn = createBackup(table, "filtered");
+ BigDecimal created = decimalAt(ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}"),
+ "/BackupDescription/BackupDetails/BackupCreationDateTime");
+
+ ddb("ListBackups", "{\"TableName\": \"BackupIT-nosuch\"}")
+ .statusCode(200)
+ .body("BackupSummaries", empty());
+ ddb("ListBackups", "{\"TableName\": \"" + table + "\", \"BackupType\": \"SYSTEM\"}")
+ .statusCode(200)
+ .body("BackupSummaries", empty());
+ ddb("ListBackups", "{\"TableName\": \"" + table + "\", \"BackupType\": \"ALL\"}")
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(backupArn));
+ ddb("ListBackups", """
+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}
+ """.formatted(table, created, created.add(BigDecimal.ONE)))
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(backupArn));
+ ddb("ListBackups", """
+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}
+ """.formatted(table, created.subtract(BigDecimal.ONE), created))
+ .statusCode(200)
+ .body("BackupSummaries", empty());
+ ddb("ListBackups", """
+ {"TableName": "%s", "TimeRangeLowerBound": %s, "TimeRangeUpperBound": %s}
+ """.formatted(table, created, created.subtract(BigDecimal.ONE)))
+ .statusCode(400)
+ .body("message", equalTo("One or more parameter values were invalid: "
+ + "Time range lower bound must be less than or equal to upper bound"));
+ }
+
+ @Test
+ void listBackupsReportsEveryViolation() {
+ ddb("ListBackups", "{\"Limit\": 0, \"BackupType\": \"BAD\"}")
+ .statusCode(400)
+ .body("message", equalTo("2 validation errors detected: Value '0' at 'limit' failed to satisfy "
+ + "constraint: Member must have value greater than or equal to 1; Value 'BAD' at "
+ + "'backupType' failed to satisfy constraint: Member must satisfy enum value set: "
+ + "[ALL, AWS_BACKUP, USER, SYSTEM]"));
+ ddb("ListBackups", "{\"Limit\": 101}")
+ .statusCode(400)
+ .body("message", equalTo("1 validation error detected: Value '101' at 'limit' failed to satisfy "
+ + "constraint: Member must have value less than or equal to 100"));
+ ddb("ListBackups", "{\"ExclusiveStartBackupArn\": \"" + "x".repeat(40) + "\"}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Backup ARN"));
+ }
+
+ @Test
+ void deleteBackupAnswersDeletedAndTheBackupIsGone() {
+ String table = "BackupIT-delete";
+ createPlainTable(table);
+ String backupArn = createBackup(table, "doomed");
+
+ ddb("DeleteBackup", "{\"BackupArn\": \"" + backupArn + "\"}")
+ .statusCode(200)
+ .body("BackupDescription.BackupDetails.BackupArn", equalTo(backupArn))
+ .body("BackupDescription.BackupDetails.BackupStatus", equalTo("DELETED"))
+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table));
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")
+ .statusCode(400)
+ .body("__type", containsString("BackupNotFoundException"));
+ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")
+ .statusCode(200)
+ .body("BackupSummaries", empty());
+ }
+
+ @Test
+ void backupOutlivesItsTable() {
+ String table = "BackupIT-outlive";
+ createPlainTable(table);
+ putItem(table, "kept");
+ String backupArn = createBackup(table, "survivor");
+ ddb("DeleteTable", "{\"TableName\": \"" + table + "\"}").statusCode(200);
+
+ ddb("ListBackups", "{\"TableName\": \"" + table + "\"}")
+ .statusCode(200)
+ .body("BackupSummaries.BackupArn", contains(backupArn));
+ ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}")
+ .statusCode(200)
+ .body("BackupDescription.SourceTableDetails.TableName", equalTo(table));
+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + table + "\"}")
+ .statusCode(200);
+ ddb("GetItem", "{\"TableName\": \"" + table + "\", \"Key\": {\"pk\": {\"S\": \"kept\"}}}")
+ .statusCode(200)
+ .body("Item.pk.S", equalTo("kept"));
+ }
+
+ @Test
+ void restoreTableFromBackupCopiesTheBackedUpItemsAndSchema() throws Exception {
+ String table = "BackupIT-restore-src";
+ String target = "BackupIT-restore-dst";
+ createIndexedTable(table);
+ ddb("UpdateTimeToLive", """
+ {"TableName": "%s", "TimeToLiveSpecification": {"Enabled": true, "AttributeName": "ttl"}}
+ """.formatted(table)).statusCode(200);
+ ddb("UpdateContinuousBackups", """
+ {"TableName": "%s", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": true}}
+ """.formatted(table)).statusCode(200);
+ putItem(table, "a", "1");
+ putItem(table, "a", "2");
+ String backupArn = createBackup(table, "restore_me");
+ putItem(table, "a", "after-backup");
+ BigDecimal backupCreated = decimalAt(ddb("DescribeBackup", "{\"BackupArn\": \"" + backupArn + "\"}"),
+ "/BackupDescription/BackupDetails/BackupCreationDateTime");
+
+ ValidatableResponse restored = ddb("RestoreTableFromBackup",
+ "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + target + "\"}");
+ assertEquals(0, backupCreated.compareTo(decimalAt(restored, "/TableDescription/RestoreSummary/RestoreDateTime")));
+ restored.statusCode(200)
+ .body("TableDescription.TableName", equalTo(target))
+ .body("TableDescription.TableArn", equalTo(TABLE_ARN_PREFIX + target))
+ .body("TableDescription.TableStatus", equalTo("CREATING"))
+ .body("TableDescription.RestoreSummary.SourceBackupArn", equalTo(backupArn))
+ .body("TableDescription.RestoreSummary.SourceTableArn", equalTo(TABLE_ARN_PREFIX + table))
+ .body("TableDescription.RestoreSummary.RestoreInProgress", equalTo(true))
+ .body("TableDescription.GlobalSecondaryIndexes[0].IndexArn",
+ equalTo(TABLE_ARN_PREFIX + target + "/index/gsi1"))
+ .body("TableDescription.LocalSecondaryIndexes[0].IndexName", equalTo("lsi1"))
+ .body("TableDescription.SSEDescription.SSEType", equalTo("KMS"));
+
+ ddb("DescribeTable", "{\"TableName\": \"" + target + "\"}")
+ .statusCode(200)
+ .body("Table.TableStatus", equalTo("ACTIVE"))
+ .body("Table", not(hasKey("RestoreSummary")))
+ .body("Table", not(hasKey("StreamSpecification")))
+ .body("Table.DeletionProtectionEnabled", equalTo(false))
+ .body("Table.TableClassSummary.TableClass", equalTo("STANDARD"))
+ .body("Table.BillingModeSummary.BillingMode", equalTo("PAY_PER_REQUEST"))
+ .body("Table.ItemCount", equalTo(2));
+ ddb("DescribeTimeToLive", "{\"TableName\": \"" + target + "\"}")
+ .statusCode(200)
+ .body("TimeToLiveDescription.TimeToLiveStatus", equalTo("DISABLED"));
+ ddb("DescribeContinuousBackups", "{\"TableName\": \"" + target + "\"}")
+ .statusCode(200)
+ .body("ContinuousBackupsDescription.PointInTimeRecoveryDescription.PointInTimeRecoveryStatus",
+ equalTo("DISABLED"));
+ ddb("ListTagsOfResource", "{\"ResourceArn\": \"" + TABLE_ARN_PREFIX + target + "\"}")
+ .statusCode(200)
+ .body("Tags", empty());
+ ddb("Scan", "{\"TableName\": \"" + target + "\"}")
+ .statusCode(200)
+ .body("Items.sk.S", containsInAnyOrder("1", "2"));
+ ddb("Query", """
+ {"TableName": "%s", "IndexName": "gsi1", "KeyConditionExpression": "g = :g",
+ "ExpressionAttributeValues": {":g": {"S": "g-1"}}}
+ """.formatted(target))
+ .statusCode(200)
+ .body("Count", equalTo(1));
+ }
+
+ @Test
+ void restoreTableFromBackupRefusesAnExistingTarget() {
+ String table = "BackupIT-exists";
+ createPlainTable(table);
+ String backupArn = createBackup(table, "exists");
+
+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + backupArn + "\", \"TargetTableName\": \"" + table + "\"}")
+ .statusCode(400)
+ .body("__type", containsString("TableAlreadyExistsException"))
+ .body("message", equalTo("Table already exists: " + table));
+ }
+
+ @Test
+ void restoreTableFromBackupChecksTheArnFirst() {
+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + "x".repeat(40) + "\", \"TargetTableName\": \"a b\"}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Backup ARN"));
+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";
+ ddb("RestoreTableFromBackup", "{\"BackupArn\": \"" + arn + "\"}")
+ .statusCode(400)
+ .body("message", equalTo("The parameter 'TableName' is required but was not present in the request"));
+ ddb("RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "a b", "BillingModeOverride": "BAD"}
+ """.formatted(arn))
+ .statusCode(400)
+ .body("message", equalTo("2 validation errors detected: Value 'BAD' at 'billingModeOverride' "
+ + "failed to satisfy constraint: Member must satisfy enum value set: "
+ + "[PROVISIONED, PAY_PER_REQUEST]; Value 'a b' at 'targetTableName' failed to satisfy "
+ + "constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"));
+ }
+
+ @Test
+ void restoreReportsEveryViolationBeforeTheBackupLookup() {
+ String arn = TABLE_ARN_PREFIX + "BackupIT-none/backup/01700000000000-abcdef12";
+ String notNull = "failed to satisfy constraint: Member must not be null";
+ String atLeastOne = "failed to satisfy constraint: Member must have value greater than or equal to 1";
+ ddb("RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "bad name!", "BillingModeOverride": "BAD",
+ "ProvisionedThroughputOverride": {},
+ "GlobalSecondaryIndexOverride": [%s]}
+ """.formatted(arn, gsi1(", \"ProvisionedThroughput\": {}").replace("\"gsi1\"", "\"x!\"")))
+ .statusCode(400)
+ .body("message", equalTo("8 validation errors detected: "
+ + "Value null at 'provisionedThroughputOverride.writeCapacityUnits' " + notNull + "; "
+ + "Value null at 'provisionedThroughputOverride.readCapacityUnits' " + notNull + "; "
+ + "Value 'BAD' at 'billingModeOverride' failed to satisfy constraint: Member must satisfy "
+ + "enum value set: [PROVISIONED, PAY_PER_REQUEST]; "
+ + "Value 'bad name!' at 'targetTableName' failed to satisfy constraint: Member must satisfy "
+ + "regular expression pattern: [a-zA-Z0-9_.-]+; "
+ + "Value 'x!' at 'globalSecondaryIndexOverride.1.member.indexName' failed to satisfy "
+ + "constraint: Member must have length greater than or equal to 3; "
+ + "Value 'x!' at 'globalSecondaryIndexOverride.1.member.indexName' failed to satisfy "
+ + "constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+; "
+ + "Value null at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."
+ + "writeCapacityUnits' " + notNull + "; "
+ + "Value null at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."
+ + "readCapacityUnits' " + notNull));
+ ddb("RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "BackupIT-none-x", "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 0}}
+ """.formatted(arn))
+ .statusCode(400)
+ .body("message", equalTo("2 validation errors detected: "
+ + "Value null at 'provisionedThroughputOverride.writeCapacityUnits' " + notNull + "; "
+ + "Value '0' at 'provisionedThroughputOverride.readCapacityUnits' " + atLeastOne));
+ ddb("RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "BackupIT-none-x", "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 0, "WriteCapacityUnits": 1},
+ "GlobalSecondaryIndexOverride": [%s]}
+ """.formatted(arn, gsi1(", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 0, "
+ + "\"WriteCapacityUnits\": 0}")))
+ .statusCode(400)
+ .body("message", equalTo("3 validation errors detected: "
+ + "Value '0' at 'provisionedThroughputOverride.readCapacityUnits' " + atLeastOne + "; "
+ + "Value '0' at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."
+ + "writeCapacityUnits' " + atLeastOne + "; "
+ + "Value '0' at 'globalSecondaryIndexOverride.1.member.provisionedThroughput."
+ + "readCapacityUnits' " + atLeastOne));
+ }
+
+ @Test
+ void restoreOverridesFollowTheBillingMode() {
+ String table = "BackupIT-billing";
+ createIndexedTable(table);
+ String backupArn = createBackup(table, "billing");
+
+ restore(backupArn, "BackupIT-billing-x", "\"BillingModeOverride\": \"PROVISIONED\"")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Must override ProvisionedThroughput if BillingMode is "
+ + "overridden to PROVISIONED"));
+ restore(backupArn, "BackupIT-billing-x", """
+ "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3}
+ """)
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Must override ProvisionedThroughput if BillingMode is "
+ + "overridden to PROVISIONED for index gsi1"));
+ restore(backupArn, "BackupIT-billing-x",
+ "\"ProvisionedThroughputOverride\": {\"ReadCapacityUnits\": 2, \"WriteCapacityUnits\": 3}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput unless "
+ + "BillingMode is PROVISIONED"));
+ restore(backupArn, "BackupIT-billing-x", """
+ "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},
+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10}
+ """)
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "
+ + "OnDemandThroughput when BillingMode is PROVISIONED"));
+ restore(backupArn, "BackupIT-billing-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(
+ ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 4, \"WriteCapacityUnits\": 5}") + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput unless "
+ + "BillingMode is PROVISIONED for index gsi1"));
+
+ restore(backupArn, "BackupIT-billing-provisioned", """
+ "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},
+ "GlobalSecondaryIndexOverride": [%s]
+ """.formatted(gsi1(", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 4, \"WriteCapacityUnits\": 5}")))
+ .statusCode(200)
+ .body("TableDescription", not(hasKey("BillingModeSummary")))
+ .body("TableDescription.ProvisionedThroughput.ReadCapacityUnits", equalTo(2))
+ .body("TableDescription.ProvisionedThroughput.WriteCapacityUnits", equalTo(3))
+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",
+ equalTo(4));
+ restore(backupArn, "BackupIT-billing-ondemand", "\"OnDemandThroughputOverride\": {\"MaxReadRequestUnits\": 10}")
+ .statusCode(200)
+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(10));
+ }
+
+ @Test
+ void restoreOnDemandOverridesMergeWithTheSourceAndFollowTheBillingMode() {
+ String table = "BackupIT-ondemand";
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "g", "AttributeType": "S"}],
+ "GlobalSecondaryIndexes": [%s],
+ "BillingMode": "PAY_PER_REQUEST",
+ "OnDemandThroughput": {"MaxReadRequestUnits": 7, "MaxWriteRequestUnits": 8}
+ }
+ """.formatted(table, gsi1(", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5, "
+ + "\"MaxWriteRequestUnits\": 6}"))).statusCode(200);
+ String backupArn = createBackup(table, "ondemand");
+ String gsiThroughput = ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 1, \"WriteCapacityUnits\": 1}";
+ String provisioned = """
+ "BillingModeOverride": "PROVISIONED",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},
+ """;
+
+ restore(backupArn, "BackupIT-ondemand-x", provisioned + "\"GlobalSecondaryIndexOverride\": [" + gsi1(
+ gsiThroughput + ", \"OnDemandThroughput\": {\"MaxWriteRequestUnits\": 5}") + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxWriteRequestUnits for "
+ + "OnDemandThroughput when BillingMode is PROVISIONED for index gsi1"));
+ restore(backupArn, "BackupIT-ondemand-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(
+ gsiThroughput + ", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}") + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "
+ + "OnDemandThroughput when ProvisionedThroughputOverride is present for index gsi1"));
+ restore(backupArn, "BackupIT-ondemand-x", """
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},
+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10}
+ """)
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "
+ + "OnDemandThroughput when ProvisionedThroughputOverride is present"));
+ restore(backupArn, "BackupIT-ondemand-x", """
+ "BillingModeOverride": "PAY_PER_REQUEST",
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3}
+ """)
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput if BillingMode "
+ + "is overridden to PAY_PER_REQUEST"));
+ restore(backupArn, "BackupIT-ondemand-x", "\"BillingModeOverride\": \"PAY_PER_REQUEST\", "
+ + "\"GlobalSecondaryIndexOverride\": [" + gsi1(gsiThroughput) + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override ProvisionedThroughput if BillingMode "
+ + "is overridden to PAY_PER_REQUEST for index gsi1"));
+
+ restore(backupArn, "BackupIT-ondemand-provisioned",
+ provisioned + "\"GlobalSecondaryIndexOverride\": [" + gsi1(gsiThroughput) + "]")
+ .statusCode(200)
+ .body("TableDescription", not(hasKey("OnDemandThroughput")))
+ .body("TableDescription.GlobalSecondaryIndexes[0]", not(hasKey("OnDemandThroughput")));
+ restore(backupArn, "BackupIT-ondemand-merged", """
+ "OnDemandThroughputOverride": {"MaxReadRequestUnits": 10},
+ "GlobalSecondaryIndexOverride": [%s]
+ """.formatted(gsi1(", \"OnDemandThroughput\": {\"MaxWriteRequestUnits\": 9}")))
+ .statusCode(200)
+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(10))
+ .body("TableDescription.OnDemandThroughput.MaxWriteRequestUnits", equalTo(8))
+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxReadRequestUnits", equalTo(5))
+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxWriteRequestUnits",
+ equalTo(9));
+ restore(backupArn, "BackupIT-ondemand-empty", """
+ "OnDemandThroughputOverride": {},
+ "GlobalSecondaryIndexOverride": [%s]
+ """.formatted(gsi1(", \"OnDemandThroughput\": {}")))
+ .statusCode(200)
+ .body("TableDescription.OnDemandThroughput.MaxReadRequestUnits", equalTo(7))
+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxWriteRequestUnits",
+ equalTo(6));
+ }
+
+ @Test
+ void restoreOfAProvisionedTableNeedsThroughputForEveryIndexOverride() {
+ String table = "BackupIT-provisioned";
+ String gsiThroughput = ", \"ProvisionedThroughput\": {\"ReadCapacityUnits\": 1, \"WriteCapacityUnits\": 1}";
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "g", "AttributeType": "S"}],
+ "GlobalSecondaryIndexes": [%s],
+ "ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}
+ }
+ """.formatted(table, gsi1(gsiThroughput))).statusCode(200);
+ String backupArn = createBackup(table, "provisioned");
+
+ restore(backupArn, "BackupIT-provisioned-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1("") + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Must specify provisioned throughput for index gsi1"));
+ restore(backupArn, "BackupIT-provisioned-x", "\"GlobalSecondaryIndexOverride\": [" + gsi1(
+ ", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}") + "]")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "
+ + "OnDemandThroughput unless BillingModeOverride is PAY_PER_REQUEST for index gsi1"));
+ restore(backupArn, "BackupIT-provisioned-x", "\"OnDemandThroughputOverride\": {\"MaxReadRequestUnits\": 10}")
+ .statusCode(400)
+ .body("message", equalTo("Invalid Request: Cannot override MaxReadRequestUnits for "
+ + "OnDemandThroughput unless BillingModeOverride is PAY_PER_REQUEST"));
+
+ restore(backupArn, "BackupIT-provisioned-kept", """
+ "ProvisionedThroughputOverride": {"ReadCapacityUnits": 2, "WriteCapacityUnits": 3},
+ "OnDemandThroughputOverride": {},
+ "GlobalSecondaryIndexOverride": [%s]
+ """.formatted(gsi1(gsiThroughput + ", \"OnDemandThroughput\": {}")))
+ .statusCode(200)
+ .body("TableDescription.ProvisionedThroughput.ReadCapacityUnits", equalTo(2))
+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",
+ equalTo(1))
+ .body("TableDescription.GlobalSecondaryIndexes[0]", not(hasKey("OnDemandThroughput")));
+ restore(backupArn, "BackupIT-provisioned-ondemand", """
+ "BillingModeOverride": "PAY_PER_REQUEST",
+ "GlobalSecondaryIndexOverride": [%s]
+ """.formatted(gsi1(", \"OnDemandThroughput\": {\"MaxReadRequestUnits\": 5}")))
+ .statusCode(200)
+ .body("TableDescription.BillingModeSummary.BillingMode", equalTo("PAY_PER_REQUEST"))
+ .body("TableDescription.GlobalSecondaryIndexes[0].ProvisionedThroughput.ReadCapacityUnits",
+ equalTo(0))
+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput.MaxReadRequestUnits", equalTo(5))
+ .body("TableDescription.GlobalSecondaryIndexes[0].OnDemandThroughput",
+ not(hasKey("MaxWriteRequestUnits")));
+ }
+
+ @Test
+ void restoreIndexOverridesMustMatchTheSourceIndexes() {
+ String table = "BackupIT-indexes";
+ createIndexedTable(table);
+ String backupArn = createBackup(table, "indexes");
+ String mismatch = "Invalid Request: Index %s does not match a secondary index that existed in the source "
+ + "table and cannot be created during the restore operation";
+
+ restore(backupArn, "BackupIT-indexes-x", """
+ "GlobalSecondaryIndexOverride": [{"IndexName": "gsi1",
+ "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], "Projection": {"ProjectionType": "KEYS_ONLY"}}]
+ """)
+ .statusCode(400)
+ .body("message", equalTo(mismatch.formatted("gsi1")));
+ restore(backupArn, "BackupIT-indexes-x", """
+ "LocalSecondaryIndexOverride": [{"IndexName": "lsi1",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],
+ "Projection": {"ProjectionType": "ALL"}}]
+ """)
+ .statusCode(400)
+ .body("message", equalTo(mismatch.formatted("lsi1")));
+
+ restore(backupArn, "BackupIT-indexes-renamed", """
+ "LocalSecondaryIndexOverride": [{"IndexName": "renamed",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],
+ "Projection": {"ProjectionType": "KEYS_ONLY"}}]
+ """)
+ .statusCode(200)
+ .body("TableDescription.LocalSecondaryIndexes.IndexName", contains("renamed"))
+ .body("TableDescription.GlobalSecondaryIndexes.IndexName", contains("gsi1"));
+ restore(backupArn, "BackupIT-indexes-dropped", """
+ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [],
+ "SSESpecificationOverride": {"Enabled": false}
+ """)
+ .statusCode(200)
+ .body("TableDescription", not(hasKey("GlobalSecondaryIndexes")))
+ .body("TableDescription", not(hasKey("LocalSecondaryIndexes")))
+ .body("TableDescription", not(hasKey("SSEDescription")))
+ .body("TableDescription.AttributeDefinitions.AttributeName", containsInAnyOrder("pk", "sk"));
+ }
+
+ private void createIndexedTable(String table) {
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}],
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "sk", "AttributeType": "S"},
+ {"AttributeName": "g", "AttributeType": "S"},
+ {"AttributeName": "l", "AttributeType": "S"}
+ ],
+ "GlobalSecondaryIndexes": [%s],
+ "LocalSecondaryIndexes": [{"IndexName": "lsi1",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}],
+ "Projection": {"ProjectionType": "KEYS_ONLY"}}],
+ "BillingMode": "PAY_PER_REQUEST",
+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"},
+ "SSESpecification": {"Enabled": true, "SSEType": "KMS"},
+ "TableClass": "STANDARD_INFREQUENT_ACCESS",
+ "DeletionProtectionEnabled": true,
+ "Tags": [{"Key": "k", "Value": "v"}]
+ }
+ """.formatted(table, gsi1(""))).statusCode(200);
+ }
+
+ private static String gsi1(String extra) {
+ return "{\"IndexName\": \"gsi1\", \"KeySchema\": [{\"AttributeName\": \"g\", \"KeyType\": \"HASH\"}], "
+ + "\"Projection\": {\"ProjectionType\": \"ALL\"}" + extra + "}";
+ }
+
+ private void createPlainTable(String table) {
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}
+ }
+ """.formatted(table)).statusCode(200);
+ }
+
+ private void putItem(String table, String pk) {
+ ddb("PutItem", "{\"TableName\": \"" + table + "\", \"Item\": {\"pk\": {\"S\": \"" + pk + "\"}}}")
+ .statusCode(200);
+ }
+
+ private void putItem(String table, String pk, String sk) {
+ ddb("PutItem", """
+ {"TableName": "%s", "Item": {"pk": {"S": "%s"}, "sk": {"S": "%s"}, "g": {"S": "g-%s"},
+ "l": {"S": "l-%s"}, "ttl": {"N": "4102444800"}}}
+ """.formatted(table, pk, sk, sk, sk)).statusCode(200);
+ }
+
+ private String createBackup(String table, String backupName) {
+ return ddb("CreateBackup", "{\"TableName\": \"" + table + "\", \"BackupName\": \"" + backupName + "\"}")
+ .statusCode(200)
+ .extract().path("BackupDetails.BackupArn");
+ }
+
+ private ValidatableResponse restore(String backupArn, String target, String overrides) {
+ return ddb("RestoreTableFromBackup", """
+ {"BackupArn": "%s", "TargetTableName": "%s", %s}
+ """.formatted(backupArn, target, overrides));
+ }
+
+ private static BigDecimal decimalAt(ValidatableResponse response, String pointer) throws Exception {
+ return EXACT.readTree(response.extract().asString()).at(pointer).decimalValue();
+ }
+
+ private ValidatableResponse ddb(String action, String body) {
+ return given()
+ .header("X-Amz-Target", "DynamoDB_20120810." + action)
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body(body)
+ .when().post("/")
+ .then();
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index 5e19bcd51d..785435c017 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.dynamodb;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -8,6 +9,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.PersistentStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
@@ -16,16 +18,19 @@ import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
+import io.github.hectorvent.floci.services.dynamodb.model.TableBackup;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -4587,4 +4592,34 @@ class DynamoDbServiceTest {
assertEquals("InterruptedByRestart", restarted.describeImport(importDesc.getImportArn()).getFailureCode());
assertEquals("FAILED", restarted.describeExport(exportDesc.getExportArn()).getExportStatus());
}
+
+ @Test
+ void backupKeepsItsSnapshotAcrossLaterWritesAndARestart(@TempDir Path dir) {
+ RegionResolver resolver = new RegionResolver("us-east-1", "000000000000");
+ StorageBackend<String, TableDefinition> tables = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> items = new InMemoryStorage<>();
+ Path backupFile = dir.resolve("dynamodb-backups.json");
+ TypeReference<Map<String, TableBackup>> backupType = new TypeReference<>() {};
+ DynamoDbService before = new DynamoDbService(tables, items, null, null,
+ new PersistentStorage<>(backupFile, backupType), resolver, null, null, null, mapper, 0, 0);
+ before.createTable("Users", List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")), 5L, 5L, "us-east-1");
+ before.putItem("Users", item("userId", "u1"), null, null, null, "us-east-1", null);
+ TableBackup backup = before.createBackup("Users", "nightly", "us-east-1");
+ before.putItem("Users", item("userId", "u2"), null, null, null, "us-east-1", null);
+ before.updateTimeToLive("Users", "expiresAt", true, "us-east-1");
+ assertFalse(before.describeBackup(backup.getBackupArn(), "us-east-1").getTable().isTtlEnabled());
+
+ PersistentStorage<String, TableBackup> reloadedBackups = new PersistentStorage<>(backupFile, backupType);
+ reloadedBackups.load();
+ DynamoDbService restarted = new DynamoDbService(tables, items, null, null, reloadedBackups, resolver,
+ null, null, null, mapper, 0, 0);
+ TableBackup reloaded = restarted.describeBackup(backup.getBackupArn(), "us-east-1");
+
+ assertEquals("nightly", reloaded.getBackupName());
+ assertEquals(backup.getCreationDateTime(), reloaded.getCreationDateTime());
+ assertEquals(List.of("u1"), reloaded.getItems().stream().map(i -> i.path("userId").path("S").asText()).toList());
+ assertEquals(1, reloaded.getTable().getItemCount());
+ assertFalse(reloaded.getTable().isTtlEnabled());
+ }
}