Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): import tables from S3 and report new exports as IN_PROGRESS (#3138)

TATSUNO “Taz” Yasuhirocbec02df0406
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
index bdc1704dfd..57d501794d 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
@@ -95,7 +95,7 @@ class DynamoDbExportTest {
@Test
@Order(1)
- void exportTableToPointInTime_returnsInProgressOrCompleted() {
+ void exportTableToPointInTime_returnsInProgress() {
ExportTableToPointInTimeResponse resp = ddb.exportTableToPointInTime(
ExportTableToPointInTimeRequest.builder()
.tableArn(tableArn)
@@ -106,7 +106,7 @@ class DynamoDbExportTest {
ExportDescription desc = resp.exportDescription();
assertThat(desc.exportArn()).isNotBlank();
- assertThat(desc.exportStatus()).isIn(ExportStatus.IN_PROGRESS, ExportStatus.COMPLETED);
+ assertThat(desc.exportStatus()).isEqualTo(ExportStatus.IN_PROGRESS);
assertThat(desc.tableArn()).isEqualTo(tableArn);
assertThat(desc.s3Bucket()).isEqualTo(BUCKET_NAME);
assertThat(desc.exportFormat()).isEqualTo(ExportFormat.DYNAMODB_JSON);
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbImportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbImportTest.java
new file mode 100644
index 0000000000..5cd217ebc6
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbImportTest.java
@@ -0,0 +1,157 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.*;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import static org.assertj.core.api.Assertions.*;
+
+@DisplayName("DynamoDB Import from S3")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbImportTest {
+
+ private static DynamoDbClient ddb;
+ private static S3Client s3;
+ private static final String TABLE_NAME = "sdk-import-test-table";
+ private static final String BUCKET_NAME = "sdk-import-test-bucket";
+ private static String importArn;
+
+ @BeforeAll
+ static void setup() {
+ ddb = TestFixtures.dynamoDbClient();
+ s3 = TestFixtures.s3Client();
+
+ try {
+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build());
+ } catch (BucketAlreadyExistsException | BucketAlreadyOwnedByYouException ignored) {}
+
+ s3.putObject(PutObjectRequest.builder().bucket(BUCKET_NAME).key("imp/data.json").build(),
+ RequestBody.fromString("{\"Item\":{\"pk\":{\"S\":\"imported1\"}}}\n{\"Item\":{\"pk\":{\"S\":\"imported2\"}}}\n"));
+ }
+
+ @AfterAll
+ static void cleanup() {
+ try {
+ ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());
+ } catch (Exception ignored) {}
+ try {
+ var objects = s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET_NAME).build());
+ for (var obj : objects.contents()) {
+ s3.deleteObject(DeleteObjectRequest.builder()
+ .bucket(BUCKET_NAME).key(obj.key()).build());
+ }
+ s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET_NAME).build());
+ } catch (Exception ignored) {}
+ if (ddb != null) {
+ ddb.close();
+ }
+ if (s3 != null) {
+ s3.close();
+ }
+ }
+
+ private static ImportTableDescription describeImport() {
+ return ddb.describeImport(DescribeImportRequest.builder().importArn(importArn).build())
+ .importTableDescription();
+ }
+
+ private static ImportTableRequest importRequest() {
+ return ImportTableRequest.builder()
+ .s3BucketSource(S3BucketSource.builder().s3Bucket(BUCKET_NAME).s3KeyPrefix("imp/").build())
+ .inputFormat(InputFormat.DYNAMODB_JSON)
+ .inputCompressionType(InputCompressionType.NONE)
+ .tableCreationParameters(TableCreationParameters.builder()
+ .tableName(TABLE_NAME)
+ .attributeDefinitions(AttributeDefinition.builder()
+ .attributeName("pk").attributeType(ScalarAttributeType.S).build())
+ .keySchema(KeySchemaElement.builder()
+ .attributeName("pk").keyType(KeyType.HASH).build())
+ .billingMode(BillingMode.PAY_PER_REQUEST)
+ .build())
+ .build();
+ }
+
+ @Test
+ @Order(1)
+ void importTable_returnsInProgress() {
+ var resp = ddb.importTable(importRequest());
+
+ var desc = resp.importTableDescription();
+ assertThat(desc.importArn()).contains("/import/");
+ assertThat(desc.importStatus()).isEqualTo(ImportStatus.IN_PROGRESS);
+ assertThat(desc.tableArn()).endsWith(":table/" + TABLE_NAME);
+ assertThat(desc.s3BucketSource().s3Bucket()).isEqualTo(BUCKET_NAME);
+ assertThat(desc.s3BucketSource().s3KeyPrefix()).isEqualTo("imp/");
+ assertThat(desc.inputFormat()).isEqualTo(InputFormat.DYNAMODB_JSON);
+ assertThat(desc.inputCompressionType()).isEqualTo(InputCompressionType.NONE);
+ assertThat(desc.tableCreationParameters().tableName()).isEqualTo(TABLE_NAME);
+ assertThat(desc.startTime()).isNotNull();
+
+ importArn = desc.importArn();
+ }
+
+ @Test
+ @Order(2)
+ void listImports_containsImport() {
+ assertThat(importArn).isNotNull();
+
+ var resp = ddb.listImports(ListImportsRequest.builder().build());
+
+ assertThat(resp.importSummaryList())
+ .extracting(ImportSummary::importArn)
+ .contains(importArn);
+ }
+
+ @Test
+ @Order(3)
+ void describeImport_reachesCompleted() throws Exception {
+ assertThat(importArn).isNotNull();
+
+ var desc = describeImport();
+ var deadline = System.currentTimeMillis() + 10_000;
+ while (desc.importStatus() == ImportStatus.IN_PROGRESS && System.currentTimeMillis() < deadline) {
+ Thread.sleep(100);
+ desc = describeImport();
+ }
+
+ assertThat(desc.importStatus())
+ .as("%s: %s", desc.failureCode(), desc.failureMessage())
+ .isEqualTo(ImportStatus.COMPLETED);
+ assertThat(desc.processedItemCount()).isEqualTo(2L);
+ assertThat(desc.importedItemCount()).isEqualTo(2L);
+ assertThat(desc.errorCount()).isZero();
+ assertThat(desc.endTime()).isNotNull();
+ }
+
+ @Test
+ @Order(4)
+ void importedTable_isActiveAndHoldsTheItems() {
+ var table = ddb.describeTable(DescribeTableRequest.builder().tableName(TABLE_NAME).build());
+ assertThat(table.table().tableStatus()).isEqualTo(TableStatus.ACTIVE);
+
+ var scan = ddb.scan(ScanRequest.builder().tableName(TABLE_NAME).build());
+ assertThat(scan.count()).isEqualTo(2);
+ assertThat(scan.items())
+ .extracting(item -> item.get("pk").s())
+ .containsExactlyInAnyOrder("imported1", "imported2");
+ }
+
+ @Test
+ @Order(5)
+ void importTable_existingTable_throwsResourceInUseException() {
+ assertThatThrownBy(() -> ddb.importTable(importRequest()))
+ .isInstanceOf(ResourceInUseException.class);
+ }
+
+ @Test
+ @Order(6)
+ void describeImport_notFound_throwsImportNotFoundException() {
+ assertThatThrownBy(() -> ddb.describeImport(DescribeImportRequest.builder()
+ .importArn("arn:aws:dynamodb:us-east-1:000000000000:table/T/import/doesnotexist")
+ .build()))
+ .isInstanceOf(ImportNotFoundException.class);
+ }
+}
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 8f6d3c0cd4..68754bdf14 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -35,6 +35,9 @@
| `ExportTableToPointInTime` | Export table data to S3 as gzip NDJSON |
| `DescribeExport` | Get export status and metadata |
| `ListExports` | List exports, optionally filtered by table ARN |
+| `ImportTable` | Create a table and load DynamoDB JSON from S3 into it |
+| `DescribeImport` | Get import status and metadata |
+| `ListImports` | List imports, optionally filtered by table ARN |
## Streams {#streams}
@@ -166,6 +169,39 @@ aws dynamodb list-exports \
The export writes to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/data/` as one or more `.json.gz` files, along with `manifest-summary.json` and `manifest-files.json`, the same layout as real AWS DynamoDB exports.
+## Import from S3
+
+Create a new table and load it from newline-delimited DynamoDB JSON objects in S3. Each line is `{"Item": {...}}`, the format an export writes:
+
+```bash
+# Upload the data
+printf '{"Item":{"userId":{"S":"u1"}}}\n{"Item":{"userId":{"S":"u2"}}}\n' > data.json
+aws s3 cp data.json s3://my-exports/imports/data.json --endpoint-url $AWS_ENDPOINT_URL
+
+# Start an import
+IMPORT_ARN=$(aws dynamodb import-table \
+ --s3-bucket-source S3Bucket=my-exports,S3KeyPrefix=imports/ \
+ --input-format DYNAMODB_JSON \
+ --input-compression-type NONE \
+ --table-creation-parameters '{"TableName":"UsersCopy","AttributeDefinitions":[{"AttributeName":"userId","AttributeType":"S"}],"KeySchema":[{"AttributeName":"userId","KeyType":"HASH"}],"BillingMode":"PAY_PER_REQUEST"}' \
+ --query ImportTableDescription.ImportArn --output text \
+ --endpoint-url $AWS_ENDPOINT_URL)
+
+# Poll until COMPLETED
+aws dynamodb describe-import \
+ --import-arn $IMPORT_ARN \
+ --query ImportTableDescription.ImportStatus \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+# List imports
+aws dynamodb list-imports --endpoint-url $AWS_ENDPOINT_URL
+```
+
+The import reads every object under the key prefix. Point it at the `data/` prefix of an export with `--input-compression-type GZIP` to load an export back. Floci does not evaluate bucket policies, so an `S3BucketOwner` that is not the caller's account fails the import with `S3AccessDenied`, as AWS does without a policy grant. The table stays in `CREATING` until the import finishes, then becomes `ACTIVE`. `DeleteTable` and `UpdateTable` return `ResourceInUseException` while the table is `CREATING`. Item calls such as `GetItem`, `PutItem`, `Query` and `Scan` return `ResourceNotFoundException` until the table is `ACTIVE`, as on AWS. A line that is not valid DynamoDB JSON or does not match the key schema is skipped and counted in `ErrorCount`. An object that cannot be read, for example a plain file under a `GZIP` import, is skipped and counted as one error. A missing bucket or an empty prefix ends the import as `FAILED` with a `FailureCode`. A reused `ClientToken` with different parameters returns `ImportConflictException`.
+
+Deviations from AWS: only `InputFormat` `DYNAMODB_JSON` with `InputCompressionType` `NONE` or `GZIP` is accepted. `CSV`, `ION` and `ZSTD` are rejected with a `ValidationException`.
+
+
## Kinesis change data capture (CDC)
When a table has an **ACTIVE** Kinesis streaming destination (see
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/RequestScopes.java b/src/main/java/io/github/hectorvent/floci/core/common/RequestScopes.java
new file mode 100644
index 0000000000..d8f575de26
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/RequestScopes.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.core.common;
+
+import io.quarkus.arc.Arc;
+
+import java.util.function.Supplier;
+
+/** Runs work that happens outside an HTTP request, such as a background worker, as a given account. */
+public final class RequestScopes {
+
+ private RequestScopes() {}
+
+ /**
+ * Account-aware storage and S3 read the account from the request context, so a worker on
+ * a fresh thread would otherwise fall back to the default account.
+ */
+ public static void runAs(String accountId, Runnable body) {
+ callAs(accountId, () -> {
+ body.run();
+ return null;
+ });
+ }
+
+ public static <T> T callAs(String accountId, Supplier<T> body) {
+ var container = Arc.container();
+ if (accountId == null || container == null || !container.isRunning()) {
+ return body.get();
+ }
+ var requestContext = container.requestContext();
+ var alreadyActive = requestContext.isActive();
+ if (!alreadyActive) {
+ requestContext.activate();
+ }
+ var ctx = container.instance(RequestContext.class).get();
+ var previousAccountId = alreadyActive ? ctx.getAccountId() : null;
+ try {
+ ctx.setAccountId(accountId);
+ return body.get();
+ } finally {
+ if (!alreadyActive) {
+ requestContext.terminate();
+ } else {
+ ctx.setAccountId(previousAccountId);
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index 8b56a83c2c..54c26275e2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -31,6 +31,7 @@ public class DynamoDbJsonHandler {
private final KinesisService kinesisService;
private final ObjectMapper objectMapper;
private final DynamoDbPartiQLHandler partiQLHandler;
+ private final Object importLock = new Object();
@Inject
public DynamoDbJsonHandler(DynamoDbService dynamoDbService, DynamoDbStreamService dynamoDbStreamService,
@@ -75,6 +76,9 @@ public class DynamoDbJsonHandler {
case "ExportTableToPointInTime" -> handleExportTable(request, region);
case "DescribeExport" -> handleDescribeExport(request, region);
case "ListExports" -> handleListExports(request, region);
+ case "ImportTable" -> handleImportTable(request, region);
+ case "DescribeImport" -> handleDescribeImport(request, region);
+ case "ListImports" -> handleListImports(request, region);
case "ExecuteStatement" -> handleExecuteStatement(request, region);
case "ExecuteTransaction" -> handleExecuteTransaction(request, region);
case "BatchExecuteStatement" -> handleBatchExecuteStatement(request, region);
@@ -86,6 +90,15 @@ public class DynamoDbJsonHandler {
}
private Response handleCreateTable(JsonNode request, String region) {
+ var table = createTableFromSpec(request, region, "ACTIVE");
+
+ var response = objectMapper.createObjectNode();
+ response.set("TableDescription", tableToNode(table));
+ return Response.ok(response).build();
+ }
+
+ /** Shared by CreateTable and ImportTable, whose TableCreationParameters use the same keys. */
+ private TableDefinition createTableFromSpec(JsonNode request, String region, String initialStatus) {
// Distinguish missing (undefined) from empty ("") for TableName
String tableNameRaw = (request.has("TableName") && !request.path("TableName").isNull())
? request.path("TableName").asText() : null;
@@ -175,6 +188,7 @@ public class DynamoDbJsonHandler {
TableDefinition table = dynamoDbService.createTable(tableName, keySchema, attrDefs,
readCapacity, writeCapacity, gsis, lsis, region);
+ table.setTableStatus(initialStatus);
table.setDeletionProtectionEnabled(deletionProtection);
@@ -230,10 +244,7 @@ public class DynamoDbJsonHandler {
// stream, billing, class, tag and SSE settings survive a restart -- stream state
// especially, since startup rebuilds streams from the persisted table.
dynamoDbService.persistTable(tableName, table, region);
-
- ObjectNode response = objectMapper.createObjectNode();
- response.set("TableDescription", tableToNode(table));
- return Response.ok(response).build();
+ return table;
}
private Response handleDeleteTable(JsonNode request, String region) {
@@ -2401,6 +2412,51 @@ public class DynamoDbJsonHandler {
return Response.ok(response).build();
}
+ private Response handleImportTable(JsonNode request, String region) {
+ var response = objectMapper.createObjectNode();
+ response.set("ImportTableDescription", objectMapper.valueToTree(findOrStartImport(request, region)));
+ return Response.ok(response).build();
+ }
+
+ /** Held across the ClientToken lookup and the table creation so identical concurrent requests share one import. */
+ private ImportTableDescription findOrStartImport(JsonNode request, String region) {
+ synchronized (importLock) {
+ var existing = dynamoDbService.validateImportRequest(request);
+ if (existing != null) {
+ return existing;
+ }
+ return dynamoDbService.startImport(request,
+ createTableFromSpec(request.path("TableCreationParameters"), region, "CREATING"), region);
+ }
+ }
+
+ private Response handleDescribeImport(JsonNode request, String region) {
+ var desc = dynamoDbService.describeImport(request.path("ImportArn").asText());
+
+ var response = objectMapper.createObjectNode();
+ response.set("ImportTableDescription", objectMapper.valueToTree(desc));
+ return Response.ok(response).build();
+ }
+
+ private Response handleListImports(JsonNode request, String region) {
+ var tableArn = request.hasNonNull("TableArn") ? request.get("TableArn").asText() : null;
+ var pageSize = request.hasNonNull("PageSize") ? request.get("PageSize").asInt() : null;
+ var nextToken = request.hasNonNull("NextToken") ? request.get("NextToken").asText() : null;
+
+ var result = dynamoDbService.listImports(tableArn, pageSize, nextToken);
+
+ var response = objectMapper.createObjectNode();
+ var summaries = objectMapper.createArrayNode();
+ for (var summary : result.importSummaryList()) {
+ summaries.add(objectMapper.valueToTree(summary));
+ }
+ response.set("ImportSummaryList", summaries);
+ if (result.nextToken() != null) {
+ response.put("NextToken", result.nextToken());
+ }
+ return Response.ok(response).build();
+ }
+
private Response handleExecuteStatement(JsonNode request, String region) {
String statement = request.path("Statement").asText();
List<JsonNode> parameters = toPartiQLParams(request.path("Parameters"));
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 0785883811..f0833a1ad8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
@@ -10,6 +11,8 @@ import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ExportDescription;
import io.github.hectorvent.floci.services.dynamodb.model.ExportSummary;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
+import io.github.hectorvent.floci.services.dynamodb.model.ImportSummary;
+import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
@@ -27,8 +30,11 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -45,6 +51,7 @@ import java.util.LinkedHashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
@@ -53,8 +60,10 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
@@ -83,6 +92,7 @@ public class DynamoDbService implements ResourceProvider {
private final StorageBackend<String, TableDefinition> tableStore;
private final StorageBackend<String, Map<String, JsonNode>> itemStore;
private final StorageBackend<String, ExportDescription> exportStore;
+ private final StorageBackend<String, ImportTableDescription> importStore;
// Items stored per table: storageKey -> Map<itemKey, item>
// itemKey is "pk" or "pk#sk" depending on table schema
private final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();
@@ -121,22 +131,24 @@ public class DynamoDbService implements ResourceProvider {
new TypeReference<Map<String, Map<String, JsonNode>>>() {}),
storageFactory.create("dynamodb", "dynamodb-exports.json",
new TypeReference<Map<String, ExportDescription>>() {}),
+ storageFactory.create("dynamodb", "dynamodb-imports.json",
+ new TypeReference<Map<String, ImportTableDescription>>() {}),
regionResolver, streamService, kinesisForwarder, s3Service, objectMapper);
}
/** Package-private constructor for testing. */
DynamoDbService(StorageBackend<String, TableDefinition> tableStore) {
- this(tableStore, null, null, new RegionResolver("us-east-1", "000000000000"), null, null, null, null);
+ this(tableStore, null, null, null, new RegionResolver("us-east-1", "000000000000"), null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore, RegionResolver regionResolver) {
- this(tableStore, null, null, regionResolver, null, null, null, null);
+ this(tableStore, null, null, null, regionResolver, null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
StorageBackend<String, Map<String, JsonNode>> itemStore,
RegionResolver regionResolver) {
- this(tableStore, itemStore, null, regionResolver, null, null, null, null);
+ this(tableStore, itemStore, null, null, regionResolver, null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
@@ -144,12 +156,13 @@ public class DynamoDbService implements ResourceProvider {
RegionResolver regionResolver,
DynamoDbStreamService streamService,
KinesisStreamingForwarder kinesisForwarder) {
- this(tableStore, itemStore, null, regionResolver, streamService, kinesisForwarder, null, null);
+ this(tableStore, itemStore, null, null, regionResolver, streamService, kinesisForwarder, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
StorageBackend<String, Map<String, JsonNode>> itemStore,
StorageBackend<String, ExportDescription> exportStore,
+ StorageBackend<String, ImportTableDescription> importStore,
RegionResolver regionResolver,
DynamoDbStreamService streamService,
KinesisStreamingForwarder kinesisForwarder,
@@ -158,12 +171,14 @@ public class DynamoDbService implements ResourceProvider {
this.tableStore = tableStore;
this.itemStore = itemStore;
this.exportStore = exportStore;
+ this.importStore = importStore;
this.regionResolver = regionResolver;
this.streamService = streamService;
this.kinesisForwarder = kinesisForwarder;
this.s3Service = s3Service;
this.objectMapper = objectMapper != null ? objectMapper : new ObjectMapper();
loadPersistedItems();
+ recoverInterruptedJobs();
}
private void loadPersistedItems() {
@@ -470,9 +485,9 @@ public class DynamoDbService implements ResourceProvider {
public void deleteTable(String tableName, String region) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- if (tableStore.get(storageKey).isEmpty()) {
- throw resourceNotFoundException(canonicalTableName);
- }
+ var table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ requireNotCreating(table);
tableStore.delete(storageKey);
itemsByTable.remove(scopedItemsKey(storageKey));
itemLocks.remove(scopedItemsKey(storageKey));
@@ -541,6 +556,8 @@ public class DynamoDbService implements ResourceProvider {
String conditionExpression,
JsonNode exprAttrNames, JsonNode exprAttrValues,
String region, String returnValuesOnConditionCheckFailure) {
+ var canonicalTableName = canonicalTableName(region, tableName);
+ requireActiveTable(regionKey(region, canonicalTableName), canonicalTableName);
return putItemInternal(tableName, item, conditionExpression, exprAttrNames, exprAttrValues,
region, returnValuesOnConditionCheckFailure, true);
}
@@ -623,8 +640,7 @@ public class DynamoDbService implements ResourceProvider {
public JsonNode getItem(String tableName, JsonNode key, String region) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
String itemKey = buildItemKey(table, key, true);
var items = currentItems(storageKey, false);
@@ -681,8 +697,7 @@ public class DynamoDbService implements ResourceProvider {
Map<String, ConcurrentSkipListMap<String, JsonNode>> stagedItems) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
String itemKey = buildItemKey(table, key, true);
@@ -778,8 +793,7 @@ public class DynamoDbService implements ResourceProvider {
Map<String, ConcurrentSkipListMap<String, JsonNode>> stagedItems) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
String itemKey = buildItemKey(table, key, true);
@@ -924,8 +938,7 @@ public class DynamoDbService implements ResourceProvider {
JsonNode exclusiveStartKey, JsonNode exprAttrNames, String region) {
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
DynamoDbAccessPath accessPath = DynamoDbAccessPath.resolve(table, indexName);
String partitionKeyValuePlaceholder = DynamoDbAccessPathValidator.validateQuery(
@@ -1083,8 +1096,7 @@ public class DynamoDbService implements ResourceProvider {
DynamoDbReservedWords.check(filterExpression, "FilterExpression");
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
DynamoDbAccessPath accessPath = DynamoDbAccessPath.resolve(table, indexName);
@@ -1205,8 +1217,7 @@ public class DynamoDbService implements ResourceProvider {
for (Map.Entry<String, List<JsonNode>> entry : requestItems.entrySet()) {
String tableName = canonicalTableName(region, entry.getKey());
String storageKey = regionKey(region, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ var table = requireActiveTable(storageKey, tableName);
Set<String> seenKeys = new HashSet<>();
for (JsonNode writeRequest : entry.getValue()) {
String itemKey;
@@ -1484,8 +1495,7 @@ public class DynamoDbService implements ResourceProvider {
}
String storageKey = regionKey(region, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ var table = requireActiveTable(storageKey, tableName);
String itemKey = buildItemKey(table, keyOrItem);
return new TransactParticipant(storageKey, itemKey);
}
@@ -1525,8 +1535,7 @@ public class DynamoDbService implements ResourceProvider {
JsonNode exprAttrValues = target.has("ExpressionAttributeValues") ? target.get("ExpressionAttributeValues") : null;
String storageKey = regionKey(region, canonicalTableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ var table = requireActiveTable(storageKey, canonicalTableName);
String itemKey = buildItemKey(table, key);
var tableItems = itemsFor(storageKey, stagedItems, false);
@@ -1573,8 +1582,7 @@ public class DynamoDbService implements ResourceProvider {
JsonNode put = transactItem.get("Put");
String tableName = canonicalTableName(region, put.path("TableName").asText());
String storageKey = regionKey(region, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ var table = requireActiveTable(storageKey, tableName);
JsonNode item = put.get("Item");
if (item == null) {
throw new AwsException("ValidationException", "Item is required for Put", 400);
@@ -1587,8 +1595,7 @@ public class DynamoDbService implements ResourceProvider {
JsonNode del = transactItem.get("Delete");
String tableName = canonicalTableName(region, del.path("TableName").asText());
String storageKey = regionKey(region, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ var table = requireActiveTable(storageKey, tableName);
JsonNode key = del.get("Key");
if (key == null) {
throw new AwsException("ValidationException", "Key is required for Delete", 400);
@@ -1598,8 +1605,7 @@ public class DynamoDbService implements ResourceProvider {
JsonNode upd = transactItem.get("Update");
String tableName = canonicalTableName(region, upd.path("TableName").asText());
String storageKey = regionKey(region, tableName);
- TableDefinition table = tableStore.get(storageKey)
- .orElseThrow(() -> resourceNotFoundException(tableName));
+ var table = requireActiveTable(storageKey, tableName);
JsonNode key = upd.get("Key");
if (key == null) {
throw new AwsException("ValidationException", "Key is required for Update", 400);
@@ -1668,6 +1674,7 @@ public class DynamoDbService implements ResourceProvider {
String storageKey = regionKey(region, canonicalTableName);
TableDefinition table = tableStore.get(storageKey)
.orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ requireNotCreating(table);
if (readCapacity != null && readCapacity <= 0) {
throw new AwsException("ValidationException",
@@ -3380,6 +3387,16 @@ public class DynamoDbService implements ResourceProvider {
"Requested resource not found: Table: " + tableName + " not found", 400);
}
+ /** Item calls treat a CREATING table as absent. AWS answers them without the table name. */
+ private TableDefinition requireActiveTable(String storageKey, String canonicalTableName) {
+ var table = tableStore.get(storageKey)
+ .orElseThrow(() -> resourceNotFoundException(canonicalTableName));
+ if ("CREATING".equals(table.getTableStatus())) {
+ throw new AwsException("ResourceNotFoundException", "Requested resource not found", 400);
+ }
+ return table;
+ }
+
public record UpdateResult(JsonNode newItem, JsonNode oldItem) {}
// scannedBytes carries the pre-filter size of the read items: DynamoDB bills a
@@ -3417,7 +3434,7 @@ public class DynamoDbService implements ResourceProvider {
.orElseThrow(() -> resourceNotFoundException(tableName));
long now = Instant.now().getEpochSecond();
- String exportId = System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");
+ var exportId = newJobId();
String exportArn = AwsArnUtils.Arn.of("dynamodb", tableRegion, regionResolver.getAccountId(), "table/" + table.getTableName() + "/export/" + exportId).toString();
ExportDescription desc = new ExportDescription();
@@ -3439,13 +3456,16 @@ public class DynamoDbService implements ResourceProvider {
exportStore.put(exportArn, desc);
}
- ExportDescription finalDesc = desc;
ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(scopedItemsKey(storageKey));
List<JsonNode> snapshot = tableItems != null
? List.copyOf(tableItems.values())
: List.of();
- Thread.ofVirtual().start(() -> runExport(finalDesc, snapshot, exportArn));
+ // The worker gets its own copy so the object serialized into this response never
+ // changes underneath the handler, and the request account so its writes land there.
+ var accountId = regionResolver.getAccountId();
+ var worker = objectMapper.convertValue(desc, ExportDescription.class);
+ Thread.ofVirtual().start(() -> RequestScopes.runAs(accountId, () -> runExport(worker, snapshot, exportArn)));
return desc;
}
@@ -3615,33 +3635,313 @@ public class DynamoDbService implements ResourceProvider {
if (exportStore == null) {
return new ListExportsResult(List.of(), null);
}
- int limit = maxResults != null ? Math.min(maxResults, 25) : 25;
-
- List<ExportDescription> all = exportStore.keys().stream()
- .map(k -> exportStore.get(k).orElse(null))
- .filter(d -> d != null)
+ var all = exportStore.scan(k -> true).stream()
.filter(d -> tableArn == null || tableArn.equals(d.getTableArn()))
- .sorted(Comparator.comparing(ExportDescription::getExportArn).reversed())
.toList();
+ requirePageSize(maxResults, "maxResults");
+ var page = pageByArn(all, ExportDescription::getExportArn, maxResults, nextToken);
+ return new ListExportsResult(page.items().stream().map(ExportSummary::new).toList(), page.nextToken());
+ }
- int startIdx = 0;
+ private record Page<T>(List<T> items, String nextToken) {}
+
+ private static final int MAX_JOB_PAGE_SIZE = 25;
+
+ private static void requirePageSize(Integer pageSize, String field) {
+ if (pageSize == null) {
+ return;
+ }
+ if (pageSize < 1) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + pageSize + "' at '" + field
+ + "' failed to satisfy constraint: Member must have value greater than or equal to 1", 400);
+ }
+ if (MAX_JOB_PAGE_SIZE < pageSize) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + pageSize + "' at '" + field
+ + "' failed to satisfy constraint: Member must have value less than or equal to "
+ + MAX_JOB_PAGE_SIZE, 400);
+ }
+ }
+
+ private static <T> Page<T> pageByArn(List<T> all, Function<T, String> arnOf, Integer pageSize, String nextToken) {
+ var limit = pageSize != null ? pageSize : MAX_JOB_PAGE_SIZE;
+ var sorted = all.stream().sorted(Comparator.comparing(arnOf).reversed()).toList();
+ var startIdx = 0;
if (nextToken != null) {
- for (int i = 0; i < all.size(); i++) {
- if (all.get(i).getExportArn().equals(nextToken)) {
+ for (var i = 0; i < sorted.size(); i++) {
+ if (nextToken.equals(arnOf.apply(sorted.get(i)))) {
startIdx = i + 1;
break;
}
}
}
+ var end = Math.min(startIdx + limit, sorted.size());
+ var newNextToken = end < sorted.size() ? arnOf.apply(sorted.get(end - 1)) : null;
+ return new Page<>(sorted.subList(startIdx, end), newNextToken);
+ }
- List<ExportDescription> page = all.subList(startIdx, Math.min(startIdx + limit, all.size()));
- String newNextToken = (startIdx + limit < all.size()) ? all.get(startIdx + limit - 1).getExportArn() : null;
+ private static String newJobId() {
+ return System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");
+ }
- List<ExportSummary> summaries = page.stream()
- .map(ExportSummary::new)
- .toList();
+ // --- Import Operations ---
+
+ /**
+ * Checks an ImportTable request before the target table exists, so a rejected request
+ * leaves nothing behind. Returns the import already started with the same ClientToken,
+ * or null when this is a new import.
+ */
+ public ImportTableDescription validateImportRequest(JsonNode request) {
+ if (request.path("S3BucketSource").path("S3Bucket").asText("").isBlank()) {
+ throw new AwsException("ValidationException", "S3BucketSource.S3Bucket is required", 400);
+ }
+ var inputFormat = request.path("InputFormat").asText("");
+ switch (inputFormat) {
+ case "DYNAMODB_JSON" -> { }
+ case "" -> throw new AwsException("ValidationException", "InputFormat is required", 400);
+ case "CSV", "ION" -> throw new AwsException("ValidationException",
+ "Unsupported InputFormat: " + inputFormat + ". Floci imports DYNAMODB_JSON only", 400);
+ default -> throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + inputFormat
+ + "' at 'inputFormat' failed to satisfy constraint: Member must satisfy enum value set: [ION, CSV, DYNAMODB_JSON]", 400);
+ }
+ var compression = request.path("InputCompressionType").asText("NONE");
+ switch (compression) {
+ case "NONE", "GZIP" -> { }
+ case "ZSTD" -> throw new AwsException("ValidationException",
+ "Unsupported InputCompressionType: ZSTD. Floci accepts NONE or GZIP", 400);
+ default -> throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + compression
+ + "' at 'inputCompressionType' failed to satisfy constraint: Member must satisfy enum value set: [GZIP, ZSTD, NONE]", 400);
+ }
+ DynamoDbTableNames.requireShortName(request.path("TableCreationParameters").path("TableName").asText(null));
+ var clientToken = request.path("ClientToken").asText(null);
+ if (clientToken != null && clientToken.isBlank()) {
+ throw new AwsException("ValidationException",
+ "1 validation error detected: Value '" + clientToken
+ + "' at 'clientToken' failed to satisfy constraint: Member must have length greater than or equal to 1", 400);
+ }
+ if (clientToken == null || importStore == null) {
+ return null;
+ }
+ var existing = importStore.scan(k -> true).stream()
+ .filter(d -> clientToken.equals(d.getClientToken()))
+ .findFirst()
+ .orElse(null);
+ if (existing != null && !sameImportParameters(existing, request)) {
+ throw new AwsException("ImportConflictException",
+ "There was a conflict when importing from the specified S3 source. This can occur when "
+ + "the current import conflicts with a previous import request that had the same client token.", 400);
+ }
+ return existing;
+ }
+
+ private static boolean sameImportParameters(ImportTableDescription existing, JsonNode request) {
+ var options = request.hasNonNull("InputFormatOptions") ? request.get("InputFormatOptions") : null;
+ return Objects.equals(existing.getS3BucketSource(), request.get("S3BucketSource"))
+ && Objects.equals(existing.getInputFormat(), request.path("InputFormat").asText())
+ && Objects.equals(existing.getInputFormatOptions(), options)
+ && Objects.equals(existing.getInputCompressionType(), request.path("InputCompressionType").asText("NONE"))
+ && Objects.equals(existing.getTableCreationParameters(), request.get("TableCreationParameters"));
+ }
+
+ public ImportTableDescription startImport(JsonNode request, TableDefinition table, String region) {
+ var tableName = table.getTableName();
+ var accountId = regionResolver.getAccountId();
+ var importArn = AwsArnUtils.Arn.of("dynamodb", region, accountId,
+ "table/" + tableName + "/import/" + newJobId()).toString();
+
+ var desc = new ImportTableDescription();
+ desc.setImportArn(importArn);
+ desc.setImportStatus("IN_PROGRESS");
+ desc.setTableArn(table.getTableArn());
+ desc.setTableId(table.getTableId());
+ desc.setClientToken(request.path("ClientToken").asText(null));
+ desc.setS3BucketSource(request.get("S3BucketSource"));
+ desc.setInputFormat(request.path("InputFormat").asText());
+ if (request.hasNonNull("InputFormatOptions")) {
+ desc.setInputFormatOptions(request.get("InputFormatOptions"));
+ }
+ desc.setInputCompressionType(request.path("InputCompressionType").asText("NONE"));
+ desc.setTableCreationParameters(request.get("TableCreationParameters"));
+ desc.setCloudWatchLogGroupArn(AwsArnUtils.Arn.of("logs", region, accountId,
+ "log-group:/aws-dynamodb/imports:*").toString());
+ desc.setStartTime(Instant.now().getEpochSecond());
+ if (importStore != null) {
+ importStore.put(importArn, desc);
+ }
+
+ // The worker gets its own copy so the object serialized into this response never
+ // changes underneath the handler, and the request account so its writes land there.
+ var worker = objectMapper.convertValue(desc, ImportTableDescription.class);
+ Thread.ofVirtual().start(() -> RequestScopes.runAs(accountId, () -> runImport(worker, tableName, region)));
+ return desc;
+ }
- return new ListExportsResult(summaries, newNextToken);
+ void runImport(ImportTableDescription desc, String tableName, String region) {
+ var source = desc.getS3BucketSource();
+ var bucket = source.path("S3Bucket").asText();
+ var prefix = source.path("S3KeyPrefix").asText("");
+ var bucketOwner = source.path("S3BucketOwner").asText(null);
+ try {
+ // AWS reads another account's bucket only when its policy grants it. floci has no bucket policies.
+ if (bucketOwner != null && !bucketOwner.equals(regionResolver.getAccountId())) {
+ throw new AwsException("AccessDenied",
+ "Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied)", 403);
+ }
+ var objects = s3Service.listObjects(bucket, prefix, null, 0);
+ if (objects.isEmpty()) {
+ failImport(desc, "S3NoSuchKey", "No objects found under s3://" + bucket + "/" + prefix);
+ } else {
+ for (var object : objects) {
+ try {
+ importObject(desc, tableName, region, bucket, object.getKey());
+ } catch (IOException | RuntimeException e) {
+ desc.setErrorCount(desc.getErrorCount() + 1);
+ LOG.warnv("Import {0} skipped object {1}: {2}", desc.getImportArn(), object.getKey(), e.getMessage());
+ }
+ }
+ desc.setImportStatus("COMPLETED");
+ }
+ } catch (AwsException e) {
+ switch (e.getErrorCode()) {
+ case "NoSuchBucket" -> failImport(desc, "S3NoSuchBucket", "The specified bucket does not exist: " + bucket);
+ case "AccessDenied" -> failImport(desc, "S3AccessDenied", e.getMessage());
+ default -> {
+ LOG.errorv(e, "Import failed: {0}", desc.getImportArn());
+ failImport(desc, "S3" + e.getErrorCode(), e.getMessage());
+ }
+ }
+ } catch (Exception e) {
+ LOG.errorv(e, "Import failed: {0}", desc.getImportArn());
+ failImport(desc, "UNKNOWN", e.getMessage());
+ }
+ persistItems(regionKey(region, tableName));
+ desc.setEndTime(Instant.now().getEpochSecond());
+ activateTable(tableName, region);
+ if (importStore != null) {
+ importStore.put(desc.getImportArn(), desc);
+ }
+ LOG.infov("Import {0} {1}: imported={2}, errors={3}", desc.getImportArn(),
+ desc.getImportStatus(), desc.getImportedItemCount(), desc.getErrorCount());
+ }
+
+ private void importObject(ImportTableDescription desc, String tableName, String region,
+ String bucket, String key) throws IOException {
+ var size = s3Service.getObjectMetadata(bucket, key, null).getSize();
+ desc.setProcessedSizeBytes(desc.getProcessedSizeBytes() + size);
+ try (var raw = s3Service.openObjectStream(bucket, key, null);
+ var reader = new BufferedReader(new InputStreamReader(decompress(desc, raw), StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (line.isBlank()) {
+ continue;
+ }
+ desc.setProcessedItemCount(desc.getProcessedItemCount() + 1);
+ try {
+ var item = objectMapper.readTree(line).path("Item");
+ if (!item.isObject()) {
+ throw new IOException("line has no Item object");
+ }
+ putItemInternal(tableName, item, null, null, null, region, "NONE", false, event -> { });
+ desc.setImportedItemCount(desc.getImportedItemCount() + 1);
+ } catch (IOException | RuntimeException e) {
+ desc.setErrorCount(desc.getErrorCount() + 1);
+ LOG.debugv("Import {0} skipped a line of {1}: {2}", desc.getImportArn(), key, e.getMessage());
+ }
+ }
+ }
+ }
+
+ private static InputStream decompress(ImportTableDescription desc, InputStream raw) throws IOException {
+ return "GZIP".equals(desc.getInputCompressionType()) ? new GZIPInputStream(raw) : raw;
+ }
+
+ private static void requireNotCreating(TableDefinition table) {
+ if ("CREATING".equals(table.getTableStatus())) {
+ throw new AwsException("ResourceInUseException",
+ "Attempt to change a resource which is still in use: Table is being created: "
+ + table.getTableName(), 400);
+ }
+ }
+
+ private static void failImport(ImportTableDescription desc, String code, String message) {
+ desc.setImportStatus("FAILED");
+ desc.setFailureCode(code);
+ desc.setFailureMessage(message);
+ }
+
+ private void activateTable(String tableName, String region) {
+ var storageKey = regionKey(region, tableName);
+ tableStore.get(storageKey).ifPresent(table -> {
+ if ("CREATING".equals(table.getTableStatus())) {
+ table.setTableStatus("ACTIVE");
+ tableStore.put(storageKey, table);
+ }
+ });
+ }
+
+ /**
+ * A job interrupted by a restart would otherwise stay IN_PROGRESS forever, since no
+ * worker survives the process. An import also leaves its table stuck in CREATING.
+ */
+ private void recoverInterruptedJobs() {
+ var now = Instant.now().getEpochSecond();
+ if (exportStore instanceof AccountAwareStorageBackend<ExportDescription> exports) {
+ for (var entry : exports.scanAllAccountEntries(k -> true)) {
+ var desc = entry.value();
+ if (!"IN_PROGRESS".equals(desc.getExportStatus())) {
+ continue;
+ }
+ desc.setExportStatus("FAILED");
+ desc.setFailureCode("InterruptedByRestart");
+ desc.setFailureMessage("The emulator restarted before the export finished");
+ desc.setEndTime(now);
+ exports.putForAccount(entry.accountId(), entry.key(), desc);
+ }
+ }
+ if (importStore instanceof AccountAwareStorageBackend<ImportTableDescription> imports) {
+ for (var entry : imports.scanAllAccountEntries(k -> true)) {
+ var desc = entry.value();
+ if (!"IN_PROGRESS".equals(desc.getImportStatus())) {
+ continue;
+ }
+ failImport(desc, "InterruptedByRestart", "The emulator restarted before the import finished");
+ desc.setEndTime(now);
+ imports.putForAccount(entry.accountId(), entry.key(), desc);
+ }
+ }
+ if (tableStore instanceof AccountAwareStorageBackend<TableDefinition> tables) {
+ for (var entry : tables.scanAllAccountEntries(k -> true)) {
+ var table = entry.value();
+ if ("CREATING".equals(table.getTableStatus())) {
+ table.setTableStatus("ACTIVE");
+ tables.putForAccount(entry.accountId(), entry.key(), table);
+ }
+ }
+ }
+ }
+
+ public ImportTableDescription describeImport(String importArn) {
+ return Optional.ofNullable(importStore)
+ .flatMap(store -> store.get(importArn))
+ .orElseThrow(() -> new AwsException("ImportNotFoundException",
+ "The specified import was not found.", 400));
+ }
+
+ public record ListImportsResult(List<ImportSummary> importSummaryList, String nextToken) {}
+
+ public ListImportsResult listImports(String tableArn, Integer pageSize, String nextToken) {
+ if (importStore == null) {
+ return new ListImportsResult(List.of(), null);
+ }
+ var all = importStore.scan(k -> true).stream()
+ .filter(d -> tableArn == null || tableArn.equals(d.getTableArn()))
+ .toList();
+ requirePageSize(pageSize, "pageSize");
+ var page = pageByArn(all, ImportTableDescription::getImportArn, pageSize, nextToken);
+ return new ListImportsResult(page.items().stream().map(ImportSummary::new).toList(), page.nextToken());
}
@Override
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportSummary.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportSummary.java
new file mode 100644
index 0000000000..5775855d0d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportSummary.java
@@ -0,0 +1,74 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ImportSummary {
+
+ @JsonProperty("ImportArn")
+ private String importArn;
+
+ @JsonProperty("ImportStatus")
+ private String importStatus;
+
+ @JsonProperty("TableArn")
+ private String tableArn;
+
+ @JsonProperty("S3BucketSource")
+ private JsonNode s3BucketSource;
+
+ @JsonProperty("CloudWatchLogGroupArn")
+ private String cloudWatchLogGroupArn;
+
+ @JsonProperty("InputFormat")
+ private String inputFormat;
+
+ @JsonProperty("StartTime")
+ private Long startTime;
+
+ @JsonProperty("EndTime")
+ private Long endTime;
+
+ public ImportSummary() {}
+
+ public ImportSummary(ImportTableDescription desc) {
+ this.importArn = desc.getImportArn();
+ this.importStatus = desc.getImportStatus();
+ this.tableArn = desc.getTableArn();
+ this.s3BucketSource = desc.getS3BucketSource();
+ this.cloudWatchLogGroupArn = desc.getCloudWatchLogGroupArn();
+ this.inputFormat = desc.getInputFormat();
+ this.startTime = desc.getStartTime();
+ this.endTime = desc.getEndTime();
+ }
+
+ public String getImportArn() { return importArn; }
+ public void setImportArn(String importArn) { this.importArn = importArn; }
+
+ public String getImportStatus() { return importStatus; }
+ public void setImportStatus(String importStatus) { this.importStatus = importStatus; }
+
+ public String getTableArn() { return tableArn; }
+ public void setTableArn(String tableArn) { this.tableArn = tableArn; }
+
+ public JsonNode getS3BucketSource() { return s3BucketSource; }
+ public void setS3BucketSource(JsonNode s3BucketSource) { this.s3BucketSource = s3BucketSource; }
+
+ public String getCloudWatchLogGroupArn() { return cloudWatchLogGroupArn; }
+ public void setCloudWatchLogGroupArn(String cloudWatchLogGroupArn) { this.cloudWatchLogGroupArn = cloudWatchLogGroupArn; }
+
+ public String getInputFormat() { return inputFormat; }
+ public void setInputFormat(String inputFormat) { this.inputFormat = inputFormat; }
+
+ public Long getStartTime() { return startTime; }
+ public void setStartTime(Long startTime) { this.startTime = startTime; }
+
+ public Long getEndTime() { return endTime; }
+ public void setEndTime(Long endTime) { this.endTime = endTime; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportTableDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportTableDescription.java
new file mode 100644
index 0000000000..320ab0e58d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ImportTableDescription.java
@@ -0,0 +1,127 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ImportTableDescription {
+
+ @JsonProperty("ImportArn")
+ private String importArn;
+
+ @JsonProperty("ImportStatus")
+ private String importStatus;
+
+ @JsonProperty("TableArn")
+ private String tableArn;
+
+ @JsonProperty("TableId")
+ private String tableId;
+
+ @JsonProperty("ClientToken")
+ private String clientToken;
+
+ @JsonProperty("S3BucketSource")
+ private JsonNode s3BucketSource;
+
+ @JsonProperty("ErrorCount")
+ private Long errorCount = 0L;
+
+ @JsonProperty("CloudWatchLogGroupArn")
+ private String cloudWatchLogGroupArn;
+
+ @JsonProperty("InputFormat")
+ private String inputFormat;
+
+ @JsonProperty("InputFormatOptions")
+ private JsonNode inputFormatOptions;
+
+ @JsonProperty("InputCompressionType")
+ private String inputCompressionType;
+
+ @JsonProperty("TableCreationParameters")
+ private JsonNode tableCreationParameters;
+
+ @JsonProperty("StartTime")
+ private Long startTime;
+
+ @JsonProperty("EndTime")
+ private Long endTime;
+
+ @JsonProperty("ProcessedSizeBytes")
+ private Long processedSizeBytes = 0L;
+
+ @JsonProperty("ProcessedItemCount")
+ private Long processedItemCount = 0L;
+
+ @JsonProperty("ImportedItemCount")
+ private Long importedItemCount = 0L;
+
+ @JsonProperty("FailureCode")
+ private String failureCode;
+
+ @JsonProperty("FailureMessage")
+ private String failureMessage;
+
+ public String getImportArn() { return importArn; }
+ public void setImportArn(String importArn) { this.importArn = importArn; }
+
+ public String getImportStatus() { return importStatus; }
+ public void setImportStatus(String importStatus) { this.importStatus = importStatus; }
+
+ public String getTableArn() { return tableArn; }
+ public void setTableArn(String tableArn) { this.tableArn = tableArn; }
+
+ public String getTableId() { return tableId; }
+ public void setTableId(String tableId) { this.tableId = tableId; }
+
+ public String getClientToken() { return clientToken; }
+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }
+
+ public JsonNode getS3BucketSource() { return s3BucketSource; }
+ public void setS3BucketSource(JsonNode s3BucketSource) { this.s3BucketSource = s3BucketSource; }
+
+ public Long getErrorCount() { return errorCount; }
+ public void setErrorCount(Long errorCount) { this.errorCount = errorCount; }
+
+ public String getCloudWatchLogGroupArn() { return cloudWatchLogGroupArn; }
+ public void setCloudWatchLogGroupArn(String cloudWatchLogGroupArn) { this.cloudWatchLogGroupArn = cloudWatchLogGroupArn; }
+
+ public String getInputFormat() { return inputFormat; }
+ public void setInputFormat(String inputFormat) { this.inputFormat = inputFormat; }
+
+ public JsonNode getInputFormatOptions() { return inputFormatOptions; }
+ public void setInputFormatOptions(JsonNode inputFormatOptions) { this.inputFormatOptions = inputFormatOptions; }
+
+ public String getInputCompressionType() { return inputCompressionType; }
+ public void setInputCompressionType(String inputCompressionType) { this.inputCompressionType = inputCompressionType; }
+
+ public JsonNode getTableCreationParameters() { return tableCreationParameters; }
+ public void setTableCreationParameters(JsonNode tableCreationParameters) { this.tableCreationParameters = tableCreationParameters; }
+
+ public Long getStartTime() { return startTime; }
+ public void setStartTime(Long startTime) { this.startTime = startTime; }
+
+ public Long getEndTime() { return endTime; }
+ public void setEndTime(Long endTime) { this.endTime = endTime; }
+
+ public Long getProcessedSizeBytes() { return processedSizeBytes; }
+ public void setProcessedSizeBytes(Long processedSizeBytes) { this.processedSizeBytes = processedSizeBytes; }
+
+ public Long getProcessedItemCount() { return processedItemCount; }
+ public void setProcessedItemCount(Long processedItemCount) { this.processedItemCount = processedItemCount; }
+
+ public Long getImportedItemCount() { return importedItemCount; }
+ public void setImportedItemCount(Long importedItemCount) { this.importedItemCount = importedItemCount; }
+
+ public String getFailureCode() { return failureCode; }
+ public void setFailureCode(String failureCode) { this.failureCode = failureCode; }
+
+ public String getFailureMessage() { return failureMessage; }
+ public void setFailureMessage(String failureMessage) { this.failureMessage = failureMessage; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
index b90d38ff80..1a3976ea28 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilder.java
@@ -143,6 +143,12 @@ public class ResourceArnBuilder {
return List.of(toDynamoDbTableArn(tableName, region, accountId));
}
}
+ if (json.path("TableCreationParameters").hasNonNull("TableName")) {
+ var tableName = json.path("TableCreationParameters").get("TableName").asText().trim();
+ if (!tableName.isEmpty()) {
+ return List.of(toDynamoDbTableArn(tableName, region, accountId));
+ }
+ }
if (json.hasNonNull("ResourceArn")) {
String resourceArn = json.get("ResourceArn").asText().trim();
if (!resourceArn.isEmpty()) {
@@ -167,6 +173,12 @@ public class ResourceArnBuilder {
return List.of(exportArn);
}
}
+ if (json.hasNonNull("ImportArn")) {
+ var importArn = json.get("ImportArn").asText().trim();
+ if (!importArn.isEmpty()) {
+ return List.of(importArn);
+ }
+ }
if (json.hasNonNull("RequestItems") && json.get("RequestItems").isObject()) {
Set<String> arns = new LinkedHashSet<>();
var fieldNames = json.get("RequestItems").fieldNames();
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
index d8b036e793..4935dba805 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
@@ -69,7 +69,7 @@ class DynamoDbExportIntegrationTest {
}
@Test
- void exportTableToPointInTime_returnsInProgressOrCompleted() {
+ void exportTableToPointInTime_returnsInProgress() {
String exportArn = given()
.header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
.contentType(DYNAMODB_CONTENT_TYPE)
@@ -85,7 +85,7 @@ class DynamoDbExportIntegrationTest {
.then()
.statusCode(200)
.body("ExportDescription.ExportArn", notNullValue())
- .body("ExportDescription.ExportStatus", oneOf("IN_PROGRESS", "COMPLETED"))
+ .body("ExportDescription.ExportStatus", equalTo("IN_PROGRESS"))
.body("ExportDescription.TableArn", equalTo(TABLE_ARN))
.body("ExportDescription.S3Bucket", equalTo(BUCKET_NAME))
.body("ExportDescription.ExportFormat", equalTo("DYNAMODB_JSON"))
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java
new file mode 100644
index 0000000000..8f0333e77c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbImportIntegrationTest.java
@@ -0,0 +1,462 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class DynamoDbImportIntegrationTest {
+
+ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String BUCKET_NAME = "import-test-bucket";
+ private static final String TABLE_ARN_PREFIX = "arn:aws:dynamodb:us-east-1:000000000000:table/";
+
+ private static boolean setupDone = false;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @BeforeEach
+ void ensureSetup() {
+ if (setupDone) {
+ return;
+ }
+ given()
+ .when().put("/" + BUCKET_NAME)
+ .then().statusCode(anyOf(equalTo(200), equalTo(409)));
+ setupDone = true;
+ }
+
+ @Test
+ void importTable_loadsNdjsonIntoNewTable() {
+ putObject("plain/data.json",
+ "{\"Item\":{\"pk\":{\"S\":\"imported1\"}}}\n{\"Item\":{\"pk\":{\"S\":\"imported2\"}}}\n");
+
+ var importArn = dynamo("ImportTable", importRequest("ImportPlain", BUCKET_NAME, "plain/", "DYNAMODB_JSON", "NONE"))
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.ImportArn", containsString("/import/"))
+ .body("ImportTableDescription.ImportStatus", equalTo("IN_PROGRESS"))
+ .body("ImportTableDescription.TableArn", equalTo(TABLE_ARN_PREFIX + "ImportPlain"))
+ .body("ImportTableDescription.TableId", notNullValue())
+ .body("ImportTableDescription.S3BucketSource.S3Bucket", equalTo(BUCKET_NAME))
+ .body("ImportTableDescription.S3BucketSource.S3KeyPrefix", equalTo("plain/"))
+ .body("ImportTableDescription.InputFormat", equalTo("DYNAMODB_JSON"))
+ .body("ImportTableDescription.InputCompressionType", equalTo("NONE"))
+ .body("ImportTableDescription.TableCreationParameters.TableName", equalTo("ImportPlain"))
+ .body("ImportTableDescription.StartTime", notNullValue())
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ dynamo("ListImports", "{}")
+ .then()
+ .statusCode(200)
+ .body("ImportSummaryList.ImportArn", hasItem(importArn));
+
+ dynamo("ListImports", "{\"TableArn\": \"" + TABLE_ARN_PREFIX + "ImportPlain\"}")
+ .then()
+ .statusCode(200)
+ .body("ImportSummaryList.ImportArn", hasItem(importArn))
+ .body("ImportSummaryList.find { it.ImportArn == '" + importArn + "' }.InputFormat", equalTo("DYNAMODB_JSON"));
+
+ dynamo("ListImports", "{\"TableArn\": \"" + TABLE_ARN_PREFIX + "SomeOtherTable\"}")
+ .then()
+ .statusCode(200)
+ .body("ImportSummaryList.ImportArn", not(hasItem(importArn)));
+
+ assertEquals("COMPLETED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.ImportArn", equalTo(importArn))
+ .body("ImportTableDescription.ImportStatus", equalTo("COMPLETED"))
+ .body("ImportTableDescription.ProcessedItemCount", equalTo(2))
+ .body("ImportTableDescription.ImportedItemCount", equalTo(2))
+ .body("ImportTableDescription.ErrorCount", equalTo(0))
+ .body("ImportTableDescription.ProcessedSizeBytes", greaterThan(0))
+ .body("ImportTableDescription.EndTime", notNullValue());
+
+ dynamo("DescribeTable", "{\"TableName\": \"ImportPlain\"}")
+ .then()
+ .statusCode(200)
+ .body("Table.TableStatus", equalTo("ACTIVE"))
+ .body("Table.ItemCount", equalTo(2));
+
+ dynamo("GetItem", "{\"TableName\": \"ImportPlain\", \"Key\": {\"pk\": {\"S\": \"imported1\"}}}")
+ .then()
+ .statusCode(200)
+ .body("Item.pk.S", equalTo("imported1"));
+ }
+
+ @Test
+ void importTable_gzip_roundTripsAnExport() {
+ createCompositeTable("RoundTripSource");
+ for (var i = 1; i <= 3; i++) {
+ dynamo("PutItem", "{\"TableName\": \"RoundTripSource\", \"Item\": {\"pk\": {\"S\": \"user-" + i
+ + "\"}, \"sk\": {\"S\": \"order-" + i + "\"}, \"total\": {\"N\": \"" + (i * 10) + "\"}}}")
+ .then().statusCode(200);
+ }
+
+ var exportArn = dynamo("ExportTableToPointInTime", """
+ {
+ "TableArn": "%s",
+ "S3Bucket": "%s",
+ "S3Prefix": "roundtrip"
+ }
+ """.formatted(TABLE_ARN_PREFIX + "RoundTripSource", BUCKET_NAME))
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ExportDescription.ExportArn");
+ assertEquals("COMPLETED", pollExport(exportArn));
+
+ var exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
+ var dataPrefix = "roundtrip/AWSDynamoDB/" + exportId + "/data/";
+
+ var importArn = dynamo("ImportTable", """
+ {
+ "S3BucketSource": {"S3Bucket": "%s", "S3KeyPrefix": "%s"},
+ "InputFormat": "DYNAMODB_JSON",
+ "InputCompressionType": "GZIP",
+ "TableCreationParameters": {
+ "TableName": "RoundTripTarget",
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "sk", "AttributeType": "S"}
+ ],
+ "KeySchema": [
+ {"AttributeName": "pk", "KeyType": "HASH"},
+ {"AttributeName": "sk", "KeyType": "RANGE"}
+ ],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ }
+ """.formatted(BUCKET_NAME, dataPrefix))
+ .then().statusCode(200)
+ .body("ImportTableDescription.ImportStatus", equalTo("IN_PROGRESS"))
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals("COMPLETED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.ImportedItemCount", equalTo(3))
+ .body("ImportTableDescription.ErrorCount", equalTo(0));
+
+ dynamo("Scan", "{\"TableName\": \"RoundTripTarget\"}")
+ .then()
+ .statusCode(200)
+ .body("Count", equalTo(3))
+ .body("Items.total.N", containsInAnyOrder("10", "20", "30"));
+ }
+
+ @Test
+ void importTable_skipsMalformedLinesAndCountsThem() {
+ putObject("bad/data.json",
+ "{\"Item\":{\"pk\":{\"S\":\"good\"}}}\nnot json at all\n{\"NoItem\":1}\n\n{\"Item\":{\"other\":{\"S\":\"missing key\"}}}\n");
+
+ var importArn = dynamo("ImportTable", importRequest("ImportBadLines", BUCKET_NAME, "bad/", "DYNAMODB_JSON", "NONE"))
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals("COMPLETED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.ProcessedItemCount", equalTo(4))
+ .body("ImportTableDescription.ImportedItemCount", equalTo(1))
+ .body("ImportTableDescription.ErrorCount", equalTo(3));
+ }
+
+ @Test
+ void importTable_missingBucket_failsWithS3NoSuchBucket() {
+ var importArn = dynamo("ImportTable", importRequest("ImportNoBucket", "no-such-import-bucket", "imp/", "DYNAMODB_JSON", "NONE"))
+ .then().statusCode(200)
+ .body("ImportTableDescription.ImportStatus", equalTo("IN_PROGRESS"))
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals("FAILED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.FailureCode", equalTo("S3NoSuchBucket"))
+ .body("ImportTableDescription.FailureMessage", containsString("no-such-import-bucket"))
+ .body("ImportTableDescription.EndTime", notNullValue());
+
+ dynamo("DescribeTable", "{\"TableName\": \"ImportNoBucket\"}")
+ .then()
+ .statusCode(200)
+ .body("Table.TableStatus", equalTo("ACTIVE"))
+ .body("Table.ItemCount", equalTo(0));
+ }
+
+ @Test
+ void importTable_bucketOfAnotherAccount_failsWithS3AccessDenied() {
+ var importArn = dynamo("ImportTable", """
+ {
+ "S3BucketSource": {"S3Bucket": "%s", "S3KeyPrefix": "plain/", "S3BucketOwner": "111111111111"},
+ "InputFormat": "DYNAMODB_JSON",
+ "TableCreationParameters": {
+ "TableName": "ImportForeignBucket",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ }
+ """.formatted(BUCKET_NAME))
+ .then().statusCode(200)
+ .body("ImportTableDescription.S3BucketSource.S3BucketOwner", equalTo("111111111111"))
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals("FAILED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.FailureCode", equalTo("S3AccessDenied"))
+ .body("ImportTableDescription.FailureMessage", startsWith("Access Denied"))
+ .body("ImportTableDescription.ProcessedItemCount", equalTo(0));
+ }
+
+ @Test
+ void importTable_emptyPrefix_fails() {
+ var importArn = dynamo("ImportTable", importRequest("ImportEmptyPrefix", BUCKET_NAME, "nothing-here/", "DYNAMODB_JSON", "NONE"))
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals("FAILED", pollUntilDone(importArn));
+
+ describeImport(importArn)
+ .then()
+ .statusCode(200)
+ .body("ImportTableDescription.FailureCode", equalTo("S3NoSuchKey"));
+ }
+
+ @Test
+ void importTable_sameClientToken_returnsTheSameImport() {
+ putObject("token/data.json", "{\"Item\":{\"pk\":{\"S\":\"one\"}}}\n");
+ var body = """
+ {
+ "ClientToken": "import-token-1",
+ "S3BucketSource": {"S3Bucket": "%s", "S3KeyPrefix": "token/"},
+ "InputFormat": "DYNAMODB_JSON",
+ "TableCreationParameters": {
+ "TableName": "ImportToken",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ }
+ """.formatted(BUCKET_NAME);
+
+ var first = dynamo("ImportTable", body)
+ .then().statusCode(200)
+ .body("ImportTableDescription.ClientToken", equalTo("import-token-1"))
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ var second = dynamo("ImportTable", body)
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+
+ assertEquals(first, second);
+ }
+
+ /** Real DynamoDB hands identical concurrent requests the same ImportArn. */
+ @Test
+ void importTable_concurrentSameClientToken_startsOneImport() throws Exception {
+ putObject("race/data.json", "{\"Item\":{\"pk\":{\"S\":\"one\"}}}\n");
+ var body = """
+ {
+ "ClientToken": "import-token-race",
+ "S3BucketSource": {"S3Bucket": "%s", "S3KeyPrefix": "race/"},
+ "InputFormat": "DYNAMODB_JSON",
+ "TableCreationParameters": {
+ "TableName": "ImportRace",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ }
+ """.formatted(BUCKET_NAME);
+ var threads = 8;
+ var startGate = new CountDownLatch(1);
+ var pool = Executors.newFixedThreadPool(threads);
+ try {
+ var futures = new ArrayList<Future<String>>();
+ for (var i = 0; i < threads; i++) {
+ futures.add(pool.submit(() -> {
+ startGate.await();
+ return dynamo("ImportTable", body)
+ .then().statusCode(200)
+ .extract().jsonPath().getString("ImportTableDescription.ImportArn");
+ }));
+ }
+ startGate.countDown();
+ var importArns = new HashSet<String>();
+ for (var future : futures) {
+ importArns.add(future.get(30, TimeUnit.SECONDS));
+ }
+ assertEquals(1, importArns.size(), importArns.toString());
+ } finally {
+ pool.shutdownNow();
+ }
+
+ dynamo("ListImports", "{\"TableArn\": \"" + TABLE_ARN_PREFIX + "ImportRace\"}")
+ .then()
+ .statusCode(200)
+ .body("ImportSummaryList", hasSize(1));
+ }
+
+ @Test
+ void importTable_existingTable_returnsResourceInUseException() {
+ createCompositeTable("ImportExisting");
+
+ dynamo("ImportTable", importRequest("ImportExisting", BUCKET_NAME, "plain/", "DYNAMODB_JSON", "NONE"))
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ResourceInUseException"))
+ .body("message", containsString("ImportExisting"));
+ }
+
+ @Test
+ void importTable_csvFormat_returnsValidationException() {
+ dynamo("ImportTable", importRequest("ImportCsv", BUCKET_NAME, "plain/", "CSV", "NONE"))
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"))
+ .body("message", containsString("CSV"));
+
+ dynamo("DescribeTable", "{\"TableName\": \"ImportCsv\"}")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ResourceNotFoundException"));
+ }
+
+ @Test
+ void importTable_zstdCompression_returnsValidationException() {
+ dynamo("ImportTable", importRequest("ImportZstd", BUCKET_NAME, "plain/", "DYNAMODB_JSON", "ZSTD"))
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"))
+ .body("message", containsString("ZSTD"));
+ }
+
+ @Test
+ void importTable_missingBucketName_returnsValidationException() {
+ dynamo("ImportTable", "{\"InputFormat\": \"DYNAMODB_JSON\", \"TableCreationParameters\": {\"TableName\": \"ImportNoSource\"}}")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"));
+ }
+
+ @Test
+ void describeImport_notFound_returnsImportNotFoundException() {
+ describeImport(TABLE_ARN_PREFIX + "T/import/doesnotexist")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ImportNotFoundException"));
+ }
+
+ // --- Helpers ---
+
+ private io.restassured.response.Response dynamo(String action, String body) {
+ return given()
+ .header("X-Amz-Target", "DynamoDB_20120810." + action)
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body(body)
+ .when().post("/");
+ }
+
+ private void putObject(String key, String body) {
+ given()
+ .header("Content-Type", "application/octet-stream")
+ .body(body.getBytes(StandardCharsets.UTF_8))
+ .when().put("/" + BUCKET_NAME + "/" + key)
+ .then().statusCode(200);
+ }
+
+ private void createCompositeTable(String tableName) {
+ dynamo("CreateTable", """
+ {
+ "TableName": "%s",
+ "KeySchema": [
+ {"AttributeName": "pk", "KeyType": "HASH"},
+ {"AttributeName": "sk", "KeyType": "RANGE"}
+ ],
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "sk", "AttributeType": "S"}
+ ],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ """.formatted(tableName))
+ .then().statusCode(200);
+ }
+
+ private String importRequest(String tableName, String bucket, String prefix, String format, String compression) {
+ return """
+ {
+ "S3BucketSource": {"S3Bucket": "%s", "S3KeyPrefix": "%s"},
+ "InputFormat": "%s",
+ "InputCompressionType": "%s",
+ "TableCreationParameters": {
+ "TableName": "%s",
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}],
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ }
+ """.formatted(bucket, prefix, format, compression, tableName);
+ }
+
+ private io.restassured.response.Response describeImport(String importArn) {
+ return dynamo("DescribeImport", "{\"ImportArn\": \"" + importArn + "\"}");
+ }
+
+ private String pollUntilDone(String importArn) {
+ return pollStatus(() -> describeImport(importArn), "ImportTableDescription.ImportStatus");
+ }
+
+ private String pollExport(String exportArn) {
+ return pollStatus(() -> dynamo("DescribeExport", "{\"ExportArn\": \"" + exportArn + "\"}"),
+ "ExportDescription.ExportStatus");
+ }
+
+ private String pollStatus(Supplier<io.restassured.response.Response> call, String statusPath) {
+ var deadline = System.currentTimeMillis() + 10_000;
+ while (System.currentTimeMillis() < deadline) {
+ var status = call.get().then().statusCode(200).extract().jsonPath().getString(statusPath);
+ if (!"IN_PROGRESS".equals(status)) {
+ return status;
+ }
+ sleep(100);
+ }
+ return "TIMEOUT";
+ }
+
+ private static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index b8016afc56..2ef6dfbb98 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -2,26 +2,35 @@ package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
+import io.github.hectorvent.floci.services.dynamodb.model.ExportDescription;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
+import io.github.hectorvent.floci.services.dynamodb.model.ImportTableDescription;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
@@ -31,6 +40,7 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
class DynamoDbServiceTest {
@@ -3754,4 +3764,353 @@ class DynamoDbServiceTest {
.put(any(), any());
assertNull(serviceWithMock.getItem("Users", item("userId", "u1"), "us-east-1"));
}
+
+ // --- ImportTable ---
+
+ private DynamoDbService serviceWithS3(S3Service s3, StorageBackend<String, ImportTableDescription> importStore) {
+ return new DynamoDbService(new InMemoryStorage<>(), new InMemoryStorage<>(), null, importStore,
+ new RegionResolver("us-east-1", "000000000000"), null, null, s3, mapper);
+ }
+
+ private void createUsersTableInCreating(DynamoDbService svc) {
+ svc.createTable("Users",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ 5L, 5L, "us-east-1").setTableStatus("CREATING");
+ }
+
+ private ImportTableDescription importDescription(String bucket, String prefix, String compression) {
+ var desc = new ImportTableDescription();
+ desc.setImportArn("arn:aws:dynamodb:us-east-1:000000000000:table/Users/import/1-abc");
+ var source = mapper.createObjectNode();
+ source.put("S3Bucket", bucket);
+ source.put("S3KeyPrefix", prefix);
+ desc.setS3BucketSource(source);
+ desc.setInputCompressionType(compression);
+ return desc;
+ }
+
+ private ObjectNode importRequest(String tableName, String inputFormat) {
+ var request = mapper.createObjectNode();
+ request.putObject("S3BucketSource").put("S3Bucket", "bucket").put("S3KeyPrefix", "imp/");
+ request.put("InputFormat", inputFormat);
+ request.putObject("TableCreationParameters").put("TableName", tableName);
+ return request;
+ }
+
+ private static byte[] gzip(String text) throws Exception {
+ var out = new ByteArrayOutputStream();
+ try (var gz = new GZIPOutputStream(out)) {
+ gz.write(text.getBytes(StandardCharsets.UTF_8));
+ }
+ return out.toByteArray();
+ }
+
+ private static S3Object s3Object(String key, byte[] data) {
+ return new S3Object("bucket", key, data, "application/octet-stream");
+ }
+
+ private static S3Service s3With(S3Object... objects) {
+ var s3 = mock(S3Service.class);
+ when(s3.listObjects("bucket", "imp/", null, 0)).thenReturn(List.of(objects));
+ for (var object : objects) {
+ when(s3.getObjectMetadata("bucket", object.getKey(), null)).thenReturn(object);
+ when(s3.openObjectStream("bucket", object.getKey(), null))
+ .thenAnswer(invocation -> new ByteArrayInputStream(object.getData()));
+ }
+ return s3;
+ }
+
+ /** Checked against real DynamoDB: every item call on a CREATING table fails this way, DescribeTable still works. */
+ @Test
+ void itemCalls_creatingTable_returnResourceNotFoundWithoutTableName() {
+ var svc = serviceWithS3(mock(S3Service.class), new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var region = "us-east-1";
+ var key = item("userId", "u1");
+ var keys = mapper.createObjectNode();
+ keys.set("Keys", mapper.createArrayNode().add(key));
+ var putRequest = mapper.createObjectNode();
+ putRequest.putObject("PutRequest").set("Item", key);
+ var transactPut = mapper.createObjectNode();
+ transactPut.putObject("Put").put("TableName", "Users").set("Item", key);
+ var transactGet = mapper.createObjectNode();
+ transactGet.putObject("Get").put("TableName", "Users").set("Key", key);
+
+ List<org.junit.jupiter.api.function.Executable> itemCalls = List.of(
+ () -> svc.getItem("Users", key, region),
+ () -> svc.putItem("Users", key, null, null, null, region, "NONE"),
+ () -> svc.updateItem("Users", key, null, "SET x = :v", null, item("v", "1"), "NONE", region),
+ () -> svc.deleteItem("Users", key, region),
+ () -> svc.query("Users", null, item("pk", "u1"), "userId = :pk", null, null, region),
+ () -> svc.scan("Users", null, null, null, null, null, null, null, region),
+ () -> svc.batchGetItem(Map.of("Users", keys), region),
+ () -> svc.batchWriteItem(Map.of("Users", List.of(putRequest)), region),
+ () -> svc.transactWriteItems(List.of(transactPut), region, null, null),
+ () -> svc.transactGetItems(List.of(transactGet), region));
+ for (var call : itemCalls) {
+ var e = assertThrows(AwsException.class, call);
+ assertEquals("ResourceNotFoundException", e.getErrorCode());
+ assertEquals("Requested resource not found", e.getMessage());
+ }
+ assertEquals("CREATING", svc.describeTable("Users", region).getTableStatus());
+ }
+
+ @Test
+ void runImport_loadsGzipLinesAndCountsBadOnes() throws Exception {
+ var object = s3Object("imp/part-0.json.gz",
+ gzip("{\"Item\":{\"userId\":{\"S\":\"u1\"}}}\nnot json\n{\"Item\":{\"userId\":{\"S\":\"u2\"}}}\n"));
+ var importStore = new InMemoryStorage<String, ImportTableDescription>();
+ var svc = serviceWithS3(s3With(object), importStore);
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "GZIP");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("COMPLETED", desc.getImportStatus());
+ assertEquals(3L, desc.getProcessedItemCount());
+ assertEquals(2L, desc.getImportedItemCount());
+ assertEquals(1L, desc.getErrorCount());
+ assertTrue(desc.getProcessedSizeBytes() > 0);
+ assertNotNull(desc.getEndTime());
+ assertEquals("COMPLETED", importStore.get(desc.getImportArn()).orElseThrow().getImportStatus());
+ assertEquals("ACTIVE", svc.describeTable("Users", "us-east-1").getTableStatus());
+ assertEquals("u2", svc.getItem("Users", item("userId", "u2"), "us-east-1").get("userId").get("S").asText());
+ }
+
+ /** Checked against real DynamoDB: another account's bucket fails this way when no policy grants access. */
+ @Test
+ void runImport_bucketOfAnotherAccount_failsWithS3AccessDeniedWithoutReadingS3() {
+ var s3 = mock(S3Service.class);
+ var svc = serviceWithS3(s3, new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "NONE");
+ ((ObjectNode) desc.getS3BucketSource()).put("S3BucketOwner", "111111111111");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("FAILED", desc.getImportStatus());
+ assertEquals("S3AccessDenied", desc.getFailureCode());
+ assertEquals("Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied)", desc.getFailureMessage());
+ verifyNoInteractions(s3);
+ assertEquals("ACTIVE", svc.describeTable("Users", "us-east-1").getTableStatus());
+ }
+
+ @Test
+ void runImport_bucketOwnerIsTheCaller_readsTheBucket() {
+ var object = s3Object("imp/data.json", "{\"Item\":{\"userId\":{\"S\":\"u1\"}}}\n".getBytes(StandardCharsets.UTF_8));
+ var svc = serviceWithS3(s3With(object), new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "NONE");
+ ((ObjectNode) desc.getS3BucketSource()).put("S3BucketOwner", "000000000000");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("COMPLETED", desc.getImportStatus());
+ assertEquals(1L, desc.getImportedItemCount());
+ }
+
+ @Test
+ void runImport_persistsLoadedItems() {
+ var object = s3Object("imp/data.json",
+ "{\"Item\":{\"userId\":{\"S\":\"u1\"}}}\n{\"Item\":{\"userId\":{\"S\":\"u2\"}}}\n".getBytes(StandardCharsets.UTF_8));
+ var itemStore = new InMemoryStorage<String, Map<String, JsonNode>>();
+ var svc = new DynamoDbService(new InMemoryStorage<>(), itemStore, null, new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), null, null, s3With(object), mapper);
+ createUsersTableInCreating(svc);
+
+ svc.runImport(importDescription("bucket", "imp/", "NONE"), "Users", "us-east-1");
+
+ assertEquals(1, itemStore.scan(k -> true).size());
+ assertEquals(2, itemStore.scan(k -> true).getFirst().size());
+ }
+
+ @Test
+ void runImport_unreadableObject_isCountedAndTheRestIsLoaded() throws Exception {
+ var data = s3Object("imp/data.json.gz", gzip("{\"Item\":{\"userId\":{\"S\":\"u1\"}}}\n"));
+ var manifest = s3Object("imp/manifest-summary.json", "{}".getBytes(StandardCharsets.UTF_8));
+ var svc = serviceWithS3(s3With(data, manifest), new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "GZIP");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("COMPLETED", desc.getImportStatus());
+ assertEquals(1L, desc.getImportedItemCount());
+ assertEquals(1L, desc.getErrorCount());
+ assertEquals("ACTIVE", svc.describeTable("Users", "us-east-1").getTableStatus());
+ }
+
+ @Test
+ void runImport_malformedAttributeValue_isCountedNotFatal() {
+ var object = s3Object("imp/data.json",
+ ("{\"Item\":{\"userId\":{\"S\":\"u1\"},\"m\":{\"M\":\"not a map\"}}}\n"
+ + "{\"Item\":{\"userId\":{\"S\":\"u2\"}}}\n").getBytes(StandardCharsets.UTF_8));
+ var svc = serviceWithS3(s3With(object), new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "NONE");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("COMPLETED", desc.getImportStatus());
+ assertEquals(1L, desc.getImportedItemCount());
+ assertEquals(1L, desc.getErrorCount());
+ }
+
+ @Test
+ void runImport_missingBucket_failsWithS3NoSuchBucket() {
+ var s3 = mock(S3Service.class);
+ when(s3.listObjects("missing", "imp/", null, 0))
+ .thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
+ var svc = serviceWithS3(s3, new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("missing", "imp/", "NONE");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("FAILED", desc.getImportStatus());
+ assertEquals("S3NoSuchBucket", desc.getFailureCode());
+ assertNotNull(desc.getEndTime());
+ assertEquals("ACTIVE", svc.describeTable("Users", "us-east-1").getTableStatus());
+ }
+
+ @Test
+ void runImport_otherS3Error_reportsAnS3FailureCode() {
+ var s3 = mock(S3Service.class);
+ when(s3.listObjects("bucket", "imp/", null, 0))
+ .thenThrow(new AwsException("AccessDenied", "Access Denied", 403));
+ var svc = serviceWithS3(s3, new InMemoryStorage<>());
+ createUsersTableInCreating(svc);
+ var desc = importDescription("bucket", "imp/", "NONE");
+
+ svc.runImport(desc, "Users", "us-east-1");
+
+ assertEquals("FAILED", desc.getImportStatus());
+ assertEquals("S3AccessDenied", desc.getFailureCode());
+ }
+
+ @Test
+ void deleteTable_whileCreating_returnsResourceInUseException() {
+ createUsersTableInCreating(service);
+
+ var e = assertThrows(AwsException.class, () -> service.deleteTable("Users", "us-east-1"));
+
+ assertEquals("ResourceInUseException", e.getErrorCode());
+ assertEquals("CREATING", service.describeTable("Users", "us-east-1").getTableStatus());
+ }
+
+ @Test
+ void updateTable_whileCreating_returnsResourceInUseException() {
+ createUsersTableInCreating(service);
+
+ var e = assertThrows(AwsException.class, () -> service.updateTable("Users", 10L, 10L, "us-east-1"));
+
+ assertEquals("ResourceInUseException", e.getErrorCode());
+ }
+
+ @Test
+ void validateImportRequest_rejectsUnsupportedFormatWithoutUnsupportedOperationWording() {
+ var e = assertThrows(AwsException.class,
+ () -> service.validateImportRequest(importRequest("Users", "CSV")));
+
+ assertEquals("ValidationException", e.getErrorCode());
+ assertTrue(e.getMessage().contains("CSV"));
+ assertFalse(e.getMessage().toLowerCase().contains("not supported"));
+ }
+
+ @Test
+ void validateImportRequest_blankClientToken_returnsValidationException() {
+ var request = importRequest("Users", "DYNAMODB_JSON");
+ request.put("ClientToken", "");
+
+ var e = assertThrows(AwsException.class, () -> service.validateImportRequest(request));
+
+ assertEquals("ValidationException", e.getErrorCode());
+ }
+
+ @Test
+ void validateImportRequest_sameClientToken_returnsExistingImport() {
+ var importStore = new InMemoryStorage<String, ImportTableDescription>();
+ var request = importRequest("Users", "DYNAMODB_JSON");
+ request.put("ClientToken", "token-1");
+ var existing = importDescription("bucket", "imp/", "NONE");
+ existing.setClientToken("token-1");
+ existing.setInputFormat("DYNAMODB_JSON");
+ existing.setTableCreationParameters(request.get("TableCreationParameters"));
+ importStore.put(existing.getImportArn(), existing);
+ var svc = serviceWithS3(mock(S3Service.class), importStore);
+
+ assertSame(existing, svc.validateImportRequest(request));
+ }
+
+ @Test
+ void validateImportRequest_sameClientTokenDifferentParameters_returnsImportConflictException() {
+ var importStore = new InMemoryStorage<String, ImportTableDescription>();
+ var existing = importDescription("bucket", "imp/", "NONE");
+ existing.setClientToken("token-1");
+ existing.setInputFormat("DYNAMODB_JSON");
+ existing.setTableCreationParameters(mapper.createObjectNode().put("TableName", "Users"));
+ importStore.put(existing.getImportArn(), existing);
+ var svc = serviceWithS3(mock(S3Service.class), importStore);
+ var request = importRequest("Other", "DYNAMODB_JSON");
+ request.put("ClientToken", "token-1");
+
+ var e = assertThrows(AwsException.class, () -> svc.validateImportRequest(request));
+
+ assertEquals("ImportConflictException", e.getErrorCode());
+ }
+
+ @Test
+ void listImports_pageSizeBelowOne_returnsValidationException() {
+ var svc = serviceWithS3(mock(S3Service.class), new InMemoryStorage<>());
+
+ var e = assertThrows(AwsException.class, () -> svc.listImports(null, 0, null));
+
+ assertEquals("ValidationException", e.getErrorCode());
+ }
+
+ @Test
+ void listImports_walksPagesByNextToken() {
+ var importStore = new InMemoryStorage<String, ImportTableDescription>();
+ for (var i = 1; i <= 3; i++) {
+ var desc = importDescription("bucket", "imp/", "NONE");
+ desc.setImportArn("arn:aws:dynamodb:us-east-1:000000000000:table/Users/import/" + i + "-abc");
+ importStore.put(desc.getImportArn(), desc);
+ }
+ var svc = serviceWithS3(mock(S3Service.class), importStore);
+
+ var first = svc.listImports(null, 2, null);
+ var second = svc.listImports(null, 2, first.nextToken());
+
+ assertEquals(2, first.importSummaryList().size());
+ assertNotNull(first.nextToken());
+ assertEquals(1, second.importSummaryList().size());
+ assertNull(second.nextToken());
+ }
+
+ @Test
+ void constructor_failsInterruptedJobsAndActivatesCreatingTables() {
+ var resolver = new RegionResolver("us-east-1", "000000000000");
+ var tables = new AccountAwareStorageBackend<TableDefinition>(new InMemoryStorage<>(), null, "000000000000");
+ var exports = new AccountAwareStorageBackend<ExportDescription>(new InMemoryStorage<>(), null, "000000000000");
+ var imports = new AccountAwareStorageBackend<ImportTableDescription>(new InMemoryStorage<>(), null, "000000000000");
+ var before = new DynamoDbService(tables, new InMemoryStorage<>(), exports, imports,
+ resolver, null, null, mock(S3Service.class), mapper);
+ createUsersTableInCreating(before);
+ var importDesc = importDescription("bucket", "imp/", "NONE");
+ importDesc.setImportStatus("IN_PROGRESS");
+ imports.put(importDesc.getImportArn(), importDesc);
+ var exportDesc = new ExportDescription();
+ exportDesc.setExportArn("arn:aws:dynamodb:us-east-1:000000000000:table/Users/export/1-abc");
+ exportDesc.setExportStatus("IN_PROGRESS");
+ exports.put(exportDesc.getExportArn(), exportDesc);
+
+ var restarted = new DynamoDbService(tables, new InMemoryStorage<>(), exports, imports,
+ resolver, null, null, mock(S3Service.class), mapper);
+
+ assertEquals("ACTIVE", restarted.describeTable("Users", "us-east-1").getTableStatus());
+ assertEquals("FAILED", restarted.describeImport(importDesc.getImportArn()).getImportStatus());
+ assertEquals("InterruptedByRestart", restarted.describeImport(importDesc.getImportArn()).getFailureCode());
+ assertEquals("FAILED", restarted.describeExport(exportDesc.getExportArn()).getExportStatus());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
index 969b11bb29..4e319f33ca 100644
--- a/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/iam/ResourceArnBuilderTest.java
@@ -196,6 +196,13 @@ class ResourceArnBuilderTest {
), arns);
}
+ @Test
+ void dynamoDbBuildsArnFromTableCreationParameters() {
+ setJsonBody("{\"TableCreationParameters\":{\"TableName\":\"ImportTarget\"},\"InputFormat\":\"DYNAMODB_JSON\"}");
+ var arn = builder.build("dynamodb", ctx, "us-east-1", "000000000000");
+ assertEquals("arn:aws:dynamodb:us-east-1:000000000000:table/ImportTarget", arn);
+ }
+
@Test
void dynamoDbReturnsWildcardWhenNoTableSpecified() {
setJsonBody("{}");