Light, fluffy, and always free - The AWS Local Emulator alternative
feat(dynamodb): implement ExportTableToPointInTime, DescribeExport, and ListExports (#653)
Hector Venturac91b74ae4ea9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.javanew file mode 100644index 0000000000..bdc1704dfd--- /dev/null+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java@@ -0,0 +1,239 @@+package com.floci.test;++import org.junit.jupiter.api.*;+import software.amazon.awssdk.core.ResponseBytes;+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;+import software.amazon.awssdk.services.dynamodb.model.*;+import software.amazon.awssdk.services.dynamodb.waiters.DynamoDbWaiter;+import software.amazon.awssdk.services.s3.S3Client;+import software.amazon.awssdk.services.s3.model.*;++import java.io.ByteArrayInputStream;+import java.io.BufferedReader;+import java.io.InputStreamReader;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;+import java.util.zip.GZIPInputStream;++import static org.assertj.core.api.Assertions.*;++@DisplayName("DynamoDB Export to S3")+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DynamoDbExportTest {++ private static DynamoDbClient ddb;+ private static S3Client s3;+ private static final String TABLE_NAME = "sdk-export-test-table";+ private static final String BUCKET_NAME = "sdk-export-test-bucket";+ private static String tableArn;+ private static String exportArn;++ @BeforeAll+ static void setup() {+ ddb = TestFixtures.dynamoDbClient();+ s3 = TestFixtures.s3Client();++ // Create S3 bucket+ try {+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build());+ } catch (BucketAlreadyExistsException | BucketAlreadyOwnedByYouException ignored) {}++ // Create DynamoDB table+ CreateTableResponse tableResp = ddb.createTable(CreateTableRequest.builder()+ .tableName(TABLE_NAME)+ .keySchema(+ KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build(),+ KeySchemaElement.builder().attributeName("sk").keyType(KeyType.RANGE).build())+ .attributeDefinitions(+ AttributeDefinition.builder().attributeName("pk")+ .attributeType(ScalarAttributeType.S).build(),+ AttributeDefinition.builder().attributeName("sk")+ .attributeType(ScalarAttributeType.S).build())+ .billingMode(BillingMode.PAY_PER_REQUEST)+ .build());+ tableArn = tableResp.tableDescription().tableArn();++ // Insert 3 items+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)+ .item(Map.of(+ "pk", AttributeValue.fromS("user-1"),+ "sk", AttributeValue.fromS("order-001"),+ "total", AttributeValue.fromN("99")))+ .build());+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)+ .item(Map.of(+ "pk", AttributeValue.fromS("user-2"),+ "sk", AttributeValue.fromS("order-002"),+ "total", AttributeValue.fromN("55")))+ .build());+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)+ .item(Map.of(+ "pk", AttributeValue.fromS("user-3"),+ "sk", AttributeValue.fromS("order-003"),+ "total", AttributeValue.fromN("150")))+ .build());+ }++ @AfterAll+ static void cleanup() {+ try {+ ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());+ } catch (Exception ignored) {}+ try {+ ListObjectsV2Response objects = s3.listObjectsV2(+ ListObjectsV2Request.builder().bucket(BUCKET_NAME).build());+ for (S3Object obj : objects.contents()) {+ s3.deleteObject(DeleteObjectRequest.builder()+ .bucket(BUCKET_NAME).key(obj.key()).build());+ }+ s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET_NAME).build());+ } catch (Exception ignored) {}+ if (ddb != null) ddb.close();+ if (s3 != null) s3.close();+ }++ @Test+ @Order(1)+ void exportTableToPointInTime_returnsInProgressOrCompleted() {+ ExportTableToPointInTimeResponse resp = ddb.exportTableToPointInTime(+ ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .s3Prefix("exports")+ .exportFormat(ExportFormat.DYNAMODB_JSON)+ .build());++ ExportDescription desc = resp.exportDescription();+ assertThat(desc.exportArn()).isNotBlank();+ assertThat(desc.exportStatus()).isIn(ExportStatus.IN_PROGRESS, ExportStatus.COMPLETED);+ assertThat(desc.tableArn()).isEqualTo(tableArn);+ assertThat(desc.s3Bucket()).isEqualTo(BUCKET_NAME);+ assertThat(desc.exportFormat()).isEqualTo(ExportFormat.DYNAMODB_JSON);+ assertThat(desc.exportType()).isEqualTo(ExportType.FULL_EXPORT);++ exportArn = desc.exportArn();+ }++ @Test+ @Order(2)+ void waitUntilExportCompleted_completesSuccessfully() {+ assertThat(exportArn).isNotNull();++ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {+ waiter.waitUntilExportCompleted(r -> r.exportArn(exportArn));+ }+ }++ @Test+ @Order(3)+ void describeExport_returnsCompletedExportWithAllFields() {+ assertThat(exportArn).isNotNull();++ DescribeExportResponse resp = ddb.describeExport(+ DescribeExportRequest.builder().exportArn(exportArn).build());++ ExportDescription desc = resp.exportDescription();+ assertThat(desc.exportStatus()).isEqualTo(ExportStatus.COMPLETED);+ assertThat(desc.tableArn()).isEqualTo(tableArn);+ assertThat(desc.s3Bucket()).isEqualTo(BUCKET_NAME);+ assertThat(desc.itemCount()).isEqualTo(3L);+ assertThat(desc.billedSizeBytes()).isGreaterThan(0L);+ assertThat(desc.exportManifest()).isNotBlank();+ assertThat(desc.startTime()).isNotNull();+ assertThat(desc.endTime()).isNotNull();+ }++ @Test+ @Order(4)+ void listExports_byTableArn_returnsExport() {+ assertThat(exportArn).isNotNull();++ ListExportsResponse resp = ddb.listExports(+ ListExportsRequest.builder().tableArn(tableArn).build());++ assertThat(resp.exportSummaries()).isNotEmpty();+ assertThat(resp.exportSummaries().stream()+ .anyMatch(s -> exportArn.equals(s.exportArn())))+ .isTrue();++ ExportSummary summary = resp.exportSummaries().stream()+ .filter(s -> exportArn.equals(s.exportArn()))+ .findFirst().orElseThrow();+ assertThat(summary.exportStatus()).isEqualTo(ExportStatus.COMPLETED);+ assertThat(summary.exportType()).isEqualTo(ExportType.FULL_EXPORT);+ }++ @Test+ @Order(5)+ void s3Objects_manifestAndDataExist() throws Exception {+ assertThat(exportArn).isNotNull();++ DescribeExportResponse descResp = ddb.describeExport(+ DescribeExportRequest.builder().exportArn(exportArn).build());+ String manifestSummaryKey = descResp.exportDescription().exportManifest();+ assertThat(manifestSummaryKey).isNotBlank();++ // Verify manifest-summary.json exists+ ResponseBytes<GetObjectResponse> manifestSummary = s3.getObjectAsBytes(+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestSummaryKey).build());+ assertThat(manifestSummary.asByteArray().length).isGreaterThan(0);++ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);+ String manifestFilesKey = "exports/AWSDynamoDB/" + exportId + "/manifest-files.json";++ ResponseBytes<GetObjectResponse> manifestFiles = s3.getObjectAsBytes(+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestFilesKey).build());+ String dataKey = new String(manifestFiles.asByteArray(), StandardCharsets.UTF_8).trim();+ assertThat(dataKey).endsWith(".json.gz");++ // Download and decompress the data file+ ResponseBytes<GetObjectResponse> dataFile = s3.getObjectAsBytes(+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());++ String ndjson = decompressGzip(dataFile.asByteArray());+ String[] lines = ndjson.split("\n");+ assertThat(lines).hasSize(3);++ for (String line : lines) {+ assertThat(line).contains("\"Item\"");+ assertThat(line).contains("\"pk\"");+ }+ }++ @Test+ @Order(6)+ void exportTableToPointInTime_invalidExportType_throwsValidationException() {+ assertThatThrownBy(() -> ddb.exportTableToPointInTime(+ ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .exportType(ExportType.INCREMENTAL_EXPORT)+ .build()))+ .isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class)+ .hasMessageContaining("not supported");+ }++ @Test+ @Order(7)+ void describeExport_notFound_throwsExportNotFoundException() {+ assertThatThrownBy(() -> ddb.describeExport(+ DescribeExportRequest.builder()+ .exportArn("arn:aws:dynamodb:us-east-1:000000000000:table/T/export/doesnotexist")+ .build()))+ .isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class);+ }++ private String decompressGzip(byte[] data) throws Exception {+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));+ BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {+ StringBuilder sb = new StringBuilder();+ String line;+ while ((line = reader.readLine()) != null) {+ if (sb.length() > 0) sb.append('\n');+ sb.append(line);+ }+ return sb.toString();+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex 155ea3035c..704bdaa75c 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -63,6 +63,9 @@ public class DynamoDbJsonHandler {case "EnableKinesisStreamingDestination" -> handleEnableKinesisStreamingDestination(request, region);case "DisableKinesisStreamingDestination" -> handleDisableKinesisStreamingDestination(request, region);case "DescribeKinesisStreamingDestination" -> handleDescribeKinesisStreamingDestination(request, region);+ case "ExportTableToPointInTime" -> handleExportTable(request, region);+ case "DescribeExport" -> handleDescribeExport(request, region);+ case "ListExports" -> handleListExports(request, region);default -> Response.status(400).entity(new AwsErrorResponse("UnknownOperationException", "Operation " + action + " is not supported.")).build();@@ -1005,4 +1008,47 @@ public class DynamoDbJsonHandler {node.set("PointInTimeRecoveryDescription", pitrNode);return node;}++ private Response handleExportTable(JsonNode request, String region) {+ Map<String, Object> params = new java.util.HashMap<>();+ request.fields().forEachRemaining(e -> params.put(e.getKey(), e.getValue().isTextual()+ ? e.getValue().asText() : e.getValue()));++ io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =+ dynamoDbService.exportTable(params, region);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("ExportDescription", objectMapper.valueToTree(desc));+ return Response.ok(response).build();+ }++ private Response handleDescribeExport(JsonNode request, String region) {+ String exportArn = request.path("ExportArn").asText();+ io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =+ dynamoDbService.describeExport(exportArn);++ ObjectNode response = objectMapper.createObjectNode();+ response.set("ExportDescription", objectMapper.valueToTree(desc));+ return Response.ok(response).build();+ }++ private Response handleListExports(JsonNode request, String region) {+ String tableArn = request.has("TableArn") ? request.get("TableArn").asText() : null;+ Integer maxResults = request.has("MaxResults") ? request.get("MaxResults").asInt() : null;+ String nextToken = request.has("NextToken") && !request.get("NextToken").isNull()+ ? request.get("NextToken").asText() : null;++ DynamoDbService.ListExportsResult result = dynamoDbService.listExports(tableArn, maxResults, nextToken);++ ObjectNode response = objectMapper.createObjectNode();+ ArrayNode summaries = objectMapper.createArrayNode();+ for (io.github.hectorvent.floci.services.dynamodb.model.ExportSummary s : result.exportSummaries()) {+ summaries.add(objectMapper.valueToTree(s));+ }+ response.set("ExportSummaries", summaries);+ if (result.nextToken() != null) {+ response.put("NextToken", result.nextToken());+ }+ return Response.ok(response).build();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex c5ca5292b6..2ddbd3f931 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -5,31 +5,44 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.ExportDescription;+import io.github.hectorvent.floci.services.dynamodb.model.ExportSummary;import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;+import io.github.hectorvent.floci.services.s3.S3Service;+import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;import java.time.Instant;import java.util.ArrayList;import java.util.Collections;import java.util.Comparator;import java.util.HashMap;+import java.util.HexFormat;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.TreeMap;+import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentSkipListMap;import java.util.concurrent.locks.ReentrantLock;import java.util.function.Supplier;+import java.util.zip.GZIPOutputStream;@ApplicationScopedpublic class DynamoDbService {@@ -38,6 +51,7 @@ public class DynamoDbService {private final StorageBackend<String, TableDefinition> tableStore;private final StorageBackend<String, Map<String, JsonNode>> itemStore;+ private final StorageBackend<String, ExportDescription> exportStore;// Items stored per table: storageKey -> Map<itemKey, item>// itemKey is "pk" or "pk#sk" depending on table schemaprivate final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();@@ -47,33 +61,39 @@ public class DynamoDbService {// not deadlock after the outer transaction already took each participant's lock.private final ConcurrentHashMap<String, ConcurrentHashMap<String, ReentrantLock>> itemLocks = new ConcurrentHashMap<>();private final RegionResolver regionResolver;+ private final ObjectMapper objectMapper;private DynamoDbStreamService streamService;private KinesisStreamingForwarder kinesisForwarder;+ private S3Service s3Service;@Injectpublic DynamoDbService(StorageFactory storageFactory, RegionResolver regionResolver,DynamoDbStreamService streamService,- KinesisStreamingForwarder kinesisForwarder) {+ KinesisStreamingForwarder kinesisForwarder,+ S3Service s3Service,+ ObjectMapper objectMapper) {this(storageFactory.create("dynamodb", "dynamodb-tables.json",new TypeReference<Map<String, TableDefinition>>() {}),storageFactory.create("dynamodb", "dynamodb-items.json",new TypeReference<Map<String, Map<String, JsonNode>>>() {}),- regionResolver, streamService, kinesisForwarder);+ storageFactory.create("dynamodb", "dynamodb-exports.json",+ new TypeReference<Map<String, ExportDescription>>() {}),+ regionResolver, streamService, kinesisForwarder, s3Service, objectMapper);}/** Package-private constructor for testing. */DynamoDbService(StorageBackend<String, TableDefinition> tableStore) {- this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null, null);+ this(tableStore, null, null, new RegionResolver("us-east-1", "000000000000"), null, null, null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore, RegionResolver regionResolver) {- this(tableStore, null, regionResolver, null, null);+ this(tableStore, null, null, regionResolver, null, null, null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore,StorageBackend<String, Map<String, JsonNode>> itemStore,RegionResolver regionResolver) {- this(tableStore, itemStore, regionResolver, null, null);+ this(tableStore, itemStore, null, regionResolver, null, null, null, null);}DynamoDbService(StorageBackend<String, TableDefinition> tableStore,@@ -81,11 +101,25 @@ public class DynamoDbService {RegionResolver regionResolver,DynamoDbStreamService streamService,KinesisStreamingForwarder kinesisForwarder) {+ this(tableStore, itemStore, null, regionResolver, streamService, kinesisForwarder, null, null);+ }++ DynamoDbService(StorageBackend<String, TableDefinition> tableStore,+ StorageBackend<String, Map<String, JsonNode>> itemStore,+ StorageBackend<String, ExportDescription> exportStore,+ RegionResolver regionResolver,+ DynamoDbStreamService streamService,+ KinesisStreamingForwarder kinesisForwarder,+ S3Service s3Service,+ ObjectMapper objectMapper) {this.tableStore = tableStore;this.itemStore = itemStore;+ this.exportStore = exportStore;this.regionResolver = regionResolver;this.streamService = streamService;this.kinesisForwarder = kinesisForwarder;+ this.s3Service = s3Service;+ this.objectMapper = objectMapper != null ? objectMapper : new ObjectMapper();loadPersistedItems();}@@ -1926,4 +1960,246 @@ public class DynamoDbService {public record UpdateResult(JsonNode newItem, JsonNode oldItem) {}public record ScanResult(List<JsonNode> items, int scannedCount, JsonNode lastEvaluatedKey) {}public record QueryResult(List<JsonNode> items, int scannedCount, JsonNode lastEvaluatedKey) {}++ // --- Export Operations ---++ public ExportDescription exportTable(Map<String, Object> request, String region) {+ String tableArn = (String) request.get("TableArn");+ String s3Bucket = (String) request.get("S3Bucket");+ String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;+ String exportFormat = request.containsKey("ExportFormat") ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";+ String exportType = request.containsKey("ExportType") ? (String) request.get("ExportType") : "FULL_EXPORT";+ String clientToken = request.containsKey("ClientToken") ? (String) request.get("ClientToken") : null;+ String s3SseAlgorithm = request.containsKey("S3SseAlgorithm") ? (String) request.get("S3SseAlgorithm") : null;+ String s3BucketOwner = request.containsKey("S3BucketOwner") ? (String) request.get("S3BucketOwner") : null;++ if ("INCREMENTAL_EXPORT".equals(exportType)) {+ throw new AwsException("ValidationException",+ "ExportType INCREMENTAL_EXPORT is not supported", 400);+ }+ if ("ION".equals(exportFormat)) {+ throw new AwsException("ValidationException",+ "ExportFormat ION is not supported", 400);+ }++ DynamoDbTableNames.ResolvedTableRef ref = DynamoDbTableNames.resolveWithRegion(tableArn, region);+ String tableName = ref.name();+ String tableRegion = ref.region() != null ? ref.region() : region;+ String storageKey = regionKey(tableRegion, tableName);++ TableDefinition table = tableStore.get(storageKey)+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "Requested resource not found: Table: " + tableName + " not found", 400));++ long now = Instant.now().getEpochSecond();+ String exportId = System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");+ String exportArn = "arn:aws:dynamodb:" + tableRegion + ":" + regionResolver.getAccountId()+ + ":table/" + table.getTableName() + "/export/" + exportId;++ ExportDescription desc = new ExportDescription();+ desc.setExportArn(exportArn);+ desc.setExportStatus("IN_PROGRESS");+ desc.setTableArn(table.getTableArn());+ desc.setTableId(table.getTableName());+ desc.setS3Bucket(s3Bucket);+ desc.setS3Prefix(s3Prefix);+ desc.setExportFormat(exportFormat);+ desc.setExportType("FULL_EXPORT");+ desc.setExportTime(now);+ desc.setStartTime(now);+ desc.setClientToken(clientToken);+ desc.setS3SseAlgorithm(s3SseAlgorithm);+ desc.setS3BucketOwner(s3BucketOwner);++ if (exportStore != null) {+ exportStore.put(exportArn, desc);+ }++ ExportDescription finalDesc = desc;+ ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(storageKey);+ List<JsonNode> snapshot = tableItems != null+ ? List.copyOf(tableItems.values())+ : List.of();++ Thread.ofVirtual().start(() -> runExport(finalDesc, snapshot, exportArn));++ return desc;+ }++ private void runExport(ExportDescription desc, List<JsonNode> snapshot, String exportArn) {+ try {+ String s3Bucket = desc.getS3Bucket();+ String s3Prefix = desc.getS3Prefix() != null ? desc.getS3Prefix() : "";+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);+ String dataFileUuid = UUID.randomUUID().toString();+ String dataKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")+ + "AWSDynamoDB/" + exportId + "/data/" + dataFileUuid + ".json.gz";+ String manifestFilesKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")+ + "AWSDynamoDB/" + exportId + "/manifest-files.json";+ String manifestSummaryKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")+ + "AWSDynamoDB/" + exportId + "/manifest-summary.json";++ byte[] gzipData = buildGzipNdjson(snapshot);++ try {+ s3Service.putObject(s3Bucket, dataKey, gzipData, "application/octet-stream", Map.of());+ } catch (AwsException e) {+ if ("NoSuchBucket".equals(e.getErrorCode())) {+ desc.setExportStatus("FAILED");+ desc.setFailureCode("S3NoSuchBucket");+ desc.setFailureMessage("The specified bucket does not exist: " + s3Bucket);+ desc.setEndTime(Instant.now().getEpochSecond());+ if (exportStore != null) {+ exportStore.put(exportArn, desc);+ }+ return;+ }+ throw e;+ }++ String md5 = computeMd5Hex(gzipData);+ String etag = md5;++ String manifestFilesContent = dataKey + "\n";+ s3Service.putObject(s3Bucket, manifestFilesKey,+ manifestFilesContent.getBytes(StandardCharsets.UTF_8),+ "application/json", Map.of());++ long billedSize = gzipData.length;+ long itemCount = snapshot.size();++ String manifestSummaryContent = buildManifestSummary(+ desc, exportId, dataKey, itemCount, billedSize, md5, etag, manifestSummaryKey);+ s3Service.putObject(s3Bucket, manifestSummaryKey,+ manifestSummaryContent.getBytes(StandardCharsets.UTF_8),+ "application/json", Map.of());++ long endTime = Instant.now().getEpochSecond();+ desc.setExportStatus("COMPLETED");+ desc.setEndTime(endTime);+ desc.setItemCount(itemCount);+ desc.setBilledSizeBytes(billedSize);+ desc.setExportManifest(manifestSummaryKey);++ if (exportStore != null) {+ exportStore.put(exportArn, desc);+ }+ LOG.infov("Export completed: {0}, items={1}", exportArn, itemCount);++ } catch (Exception e) {+ LOG.errorv(e, "Export failed: {0}", exportArn);+ desc.setExportStatus("FAILED");+ desc.setFailureCode("UNKNOWN");+ desc.setFailureMessage(e.getMessage());+ desc.setEndTime(Instant.now().getEpochSecond());+ if (exportStore != null) {+ exportStore.put(exportArn, desc);+ }+ }+ }++ private byte[] buildGzipNdjson(List<JsonNode> items) throws IOException {+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {+ for (JsonNode item : items) {+ ObjectNode line = objectMapper.createObjectNode();+ line.set("Item", item);+ byte[] lineBytes = objectMapper.writeValueAsBytes(line);+ gzip.write(lineBytes);+ gzip.write('\n');+ }+ }+ return baos.toByteArray();+ }++ private String computeMd5Hex(byte[] data) {+ try {+ MessageDigest md = MessageDigest.getInstance("MD5");+ byte[] digest = md.digest(data);+ return HexFormat.of().formatHex(digest);+ } catch (NoSuchAlgorithmException e) {+ return "unknown";+ }+ }++ private String buildManifestSummary(ExportDescription desc, String exportId,+ String dataKey, long itemCount, long billedSize,+ String md5, String etag, String manifestSummaryKey) {+ try {+ com.fasterxml.jackson.databind.node.ObjectNode root = objectMapper.createObjectNode();+ root.put("version", "2020-06-30");+ root.put("exportArn", desc.getExportArn());+ root.put("startTime", Instant.ofEpochSecond(desc.getStartTime()).toString());+ root.put("endTime", Instant.now().toString());+ root.put("tableArn", desc.getTableArn());+ root.put("tableId", desc.getTableId());+ root.put("exportTime", Instant.ofEpochSecond(desc.getExportTime()).toString());+ root.put("s3Bucket", desc.getS3Bucket());+ root.putNull("s3Prefix");+ if (desc.getS3Prefix() != null) {+ root.put("s3Prefix", desc.getS3Prefix());+ }+ root.put("s3SseAlgorithm", desc.getS3SseAlgorithm() != null ? desc.getS3SseAlgorithm() : "AES256");+ root.putNull("s3SseKmsKeyId");+ root.put("exportFormat", desc.getExportFormat());+ root.put("billedSizeBytes", billedSize);+ root.put("itemCount", itemCount);++ com.fasterxml.jackson.databind.node.ArrayNode outputFiles = root.putArray("outputFiles");+ com.fasterxml.jackson.databind.node.ObjectNode fileEntry = outputFiles.addObject();+ fileEntry.put("itemCount", itemCount);+ fileEntry.put("md5Checksum", md5);+ fileEntry.put("etag", etag);+ fileEntry.put("dataFileS3Key", dataKey);++ return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(root);+ } catch (JsonProcessingException e) {+ return "{}";+ }+ }++ public ExportDescription describeExport(String exportArn) {+ if (exportStore == null) {+ throw new AwsException("ExportNotFoundException",+ "Export not found: " + exportArn, 400);+ }+ return exportStore.get(exportArn)+ .orElseThrow(() -> new AwsException("ExportNotFoundException",+ "Export not found: " + exportArn, 400));+ }++ public record ListExportsResult(List<ExportSummary> exportSummaries, String nextToken) {}++ public ListExportsResult listExports(String tableArn, Integer maxResults, String nextToken) {+ if (exportStore == null) {+ return new ListExportsResult(List.of(), null);+ }+ int limit = maxResults != null ? Math.min(maxResults, 25) : 25;++ List<ExportDescription> all = exportStore.keys().stream()+ .map(k -> exportStore.get(k).orElse(null))+ .filter(d -> d != null)+ .filter(d -> tableArn == null || tableArn.equals(d.getTableArn()))+ .sorted(Comparator.comparing(ExportDescription::getExportArn).reversed())+ .toList();++ int startIdx = 0;+ if (nextToken != null) {+ for (int i = 0; i < all.size(); i++) {+ if (all.get(i).getExportArn().equals(nextToken)) {+ startIdx = i + 1;+ break;+ }+ }+ }++ List<ExportDescription> page = all.subList(startIdx, Math.min(startIdx + limit, all.size()));+ String newNextToken = (startIdx + limit < all.size()) ? all.get(startIdx + limit - 1).getExportArn() : null;++ List<ExportSummary> summaries = page.stream()+ .map(ExportSummary::new)+ .toList();++ return new ListExportsResult(summaries, newNextToken);+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.javanew file mode 100644index 0000000000..4ae924b18d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java@@ -0,0 +1,126 @@+package io.github.hectorvent.floci.services.dynamodb.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ExportDescription {++ @JsonProperty("ExportArn")+ private String exportArn;++ @JsonProperty("ExportStatus")+ private String exportStatus;++ @JsonProperty("TableArn")+ private String tableArn;++ @JsonProperty("TableId")+ private String tableId;++ @JsonProperty("S3Bucket")+ private String s3Bucket;++ @JsonProperty("S3Prefix")+ private String s3Prefix;++ @JsonProperty("ExportFormat")+ private String exportFormat;++ @JsonProperty("ExportType")+ private String exportType;++ @JsonProperty("ExportTime")+ private Long exportTime;++ @JsonProperty("StartTime")+ private Long startTime;++ @JsonProperty("EndTime")+ private Long endTime;++ @JsonProperty("ItemCount")+ private Long itemCount;++ @JsonProperty("BilledSizeBytes")+ private Long billedSizeBytes;++ @JsonProperty("ExportManifest")+ private String exportManifest;++ @JsonProperty("ClientToken")+ private String clientToken;++ @JsonProperty("S3SseAlgorithm")+ private String s3SseAlgorithm;++ @JsonProperty("S3BucketOwner")+ private String s3BucketOwner;++ @JsonProperty("FailureCode")+ private String failureCode;++ @JsonProperty("FailureMessage")+ private String failureMessage;++ public String getExportArn() { return exportArn; }+ public void setExportArn(String exportArn) { this.exportArn = exportArn; }++ public String getExportStatus() { return exportStatus; }+ public void setExportStatus(String exportStatus) { this.exportStatus = exportStatus; }++ public String getTableArn() { return tableArn; }+ public void setTableArn(String tableArn) { this.tableArn = tableArn; }++ public String getTableId() { return tableId; }+ public void setTableId(String tableId) { this.tableId = tableId; }++ public String getS3Bucket() { return s3Bucket; }+ public void setS3Bucket(String s3Bucket) { this.s3Bucket = s3Bucket; }++ public String getS3Prefix() { return s3Prefix; }+ public void setS3Prefix(String s3Prefix) { this.s3Prefix = s3Prefix; }++ public String getExportFormat() { return exportFormat; }+ public void setExportFormat(String exportFormat) { this.exportFormat = exportFormat; }++ public String getExportType() { return exportType; }+ public void setExportType(String exportType) { this.exportType = exportType; }++ public Long getExportTime() { return exportTime; }+ public void setExportTime(Long exportTime) { this.exportTime = exportTime; }++ public Long getStartTime() { return startTime; }+ public void setStartTime(Long startTime) { this.startTime = startTime; }++ public Long getEndTime() { return endTime; }+ public void setEndTime(Long endTime) { this.endTime = endTime; }++ public Long getItemCount() { return itemCount; }+ public void setItemCount(Long itemCount) { this.itemCount = itemCount; }++ public Long getBilledSizeBytes() { return billedSizeBytes; }+ public void setBilledSizeBytes(Long billedSizeBytes) { this.billedSizeBytes = billedSizeBytes; }++ public String getExportManifest() { return exportManifest; }+ public void setExportManifest(String exportManifest) { this.exportManifest = exportManifest; }++ public String getClientToken() { return clientToken; }+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }++ public String getS3SseAlgorithm() { return s3SseAlgorithm; }+ public void setS3SseAlgorithm(String s3SseAlgorithm) { this.s3SseAlgorithm = s3SseAlgorithm; }++ public String getS3BucketOwner() { return s3BucketOwner; }+ public void setS3BucketOwner(String s3BucketOwner) { this.s3BucketOwner = s3BucketOwner; }++ public String getFailureCode() { return failureCode; }+ public void setFailureCode(String failureCode) { this.failureCode = failureCode; }++ public String getFailureMessage() { return failureMessage; }+ public void setFailureMessage(String failureMessage) { this.failureMessage = failureMessage; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.javanew file mode 100644index 0000000000..7eedbfbeaa--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.java@@ -0,0 +1,38 @@+package io.github.hectorvent.floci.services.dynamodb.model;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import com.fasterxml.jackson.annotation.JsonInclude;+import com.fasterxml.jackson.annotation.JsonProperty;+import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+@JsonIgnoreProperties(ignoreUnknown = true)+@JsonInclude(JsonInclude.Include.NON_NULL)+public class ExportSummary {++ @JsonProperty("ExportArn")+ private String exportArn;++ @JsonProperty("ExportStatus")+ private String exportStatus;++ @JsonProperty("ExportType")+ private String exportType;++ public ExportSummary() {}++ public ExportSummary(ExportDescription desc) {+ this.exportArn = desc.getExportArn();+ this.exportStatus = desc.getExportStatus();+ this.exportType = desc.getExportType();+ }++ public String getExportArn() { return exportArn; }+ public void setExportArn(String exportArn) { this.exportArn = exportArn; }++ public String getExportStatus() { return exportStatus; }+ public void setExportStatus(String exportStatus) { this.exportStatus = exportStatus; }++ public String getExportType() { return exportType; }+ public void setExportType(String exportType) { this.exportType = exportType; }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.javanew file mode 100644index 0000000000..0cbce5eb55--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java@@ -0,0 +1,315 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.io.InputStreamReader;+import java.io.BufferedReader;+import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.zip.GZIPInputStream;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.*;+import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class DynamoDbExportIntegrationTest {++ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";+ private static final String TABLE_NAME = "ExportTestTable";+ private static final String BUCKET_NAME = "export-test-bucket";+ private static final String TABLE_ARN =+ "arn:aws:dynamodb:us-east-1:000000000000:table/" + TABLE_NAME;++ private static boolean setupDone = false;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @BeforeEach+ void ensureSetup() {+ if (setupDone) return;++ // Create S3 bucket+ given()+ .when().put("/" + BUCKET_NAME)+ .then().statusCode(anyOf(equalTo(200), equalTo(409)));++ // Create table+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [+ {"AttributeName": "pk", "KeyType": "HASH"},+ {"AttributeName": "sk", "KeyType": "RANGE"}+ ],+ "AttributeDefinitions": [+ {"AttributeName": "pk", "AttributeType": "S"},+ {"AttributeName": "sk", "AttributeType": "S"}+ ]+ }+ """.formatted(TABLE_NAME))+ .when().post("/").then().statusCode(200);++ // Insert 3 items+ putItem("{\"pk\": {\"S\": \"user-1\"}, \"sk\": {\"S\": \"order-001\"}, \"total\": {\"N\": \"99\"}}");+ putItem("{\"pk\": {\"S\": \"user-2\"}, \"sk\": {\"S\": \"order-002\"}, \"total\": {\"N\": \"55\"}}");+ putItem("{\"pk\": {\"S\": \"user-3\"}, \"sk\": {\"S\": \"order-003\"}, \"total\": {\"N\": \"150\"}}");++ setupDone = true;+ }++ @Test+ void exportTableToPointInTime_returnsInProgressOrCompleted() {+ String exportArn = given()+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableArn": "%s",+ "S3Bucket": "%s",+ "S3Prefix": "exports",+ "ExportFormat": "DYNAMODB_JSON"+ }+ """.formatted(TABLE_ARN, BUCKET_NAME))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("ExportDescription.ExportArn", notNullValue())+ .body("ExportDescription.ExportStatus", oneOf("IN_PROGRESS", "COMPLETED"))+ .body("ExportDescription.TableArn", equalTo(TABLE_ARN))+ .body("ExportDescription.S3Bucket", equalTo(BUCKET_NAME))+ .body("ExportDescription.ExportFormat", equalTo("DYNAMODB_JSON"))+ .body("ExportDescription.ExportType", equalTo("FULL_EXPORT"))+ .extract().path("ExportDescription.ExportArn");++ assertNotNull(exportArn);+ assertTrue(exportArn.contains("/export/"), "ExportArn should contain /export/ segment");++ // Poll DescribeExport until COMPLETED (max 10s)+ String status = pollUntilCompleted(exportArn, 10_000);+ assertEquals("COMPLETED", status, "Export should complete within 10 seconds");+ }++ @Test+ void describeExport_returnsCompletedExportWithS3Manifest() throws Exception {+ String exportArn = startExport("exports-describe");+ String status = pollUntilCompleted(exportArn, 10_000);+ assertEquals("COMPLETED", status);++ // Verify ExportManifest is set+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"ExportArn\": \"" + exportArn + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("ExportDescription.ExportStatus", equalTo("COMPLETED"))+ .body("ExportDescription.ItemCount", equalTo(3))+ .body("ExportDescription.BilledSizeBytes", greaterThan(0))+ .body("ExportDescription.ExportManifest", notNullValue());+ }++ @Test+ void listExports_returnsCompletedExport() throws Exception {+ String exportArn = startExport("exports-list");+ pollUntilCompleted(exportArn, 10_000);++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.ListExports")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableArn\": \"" + TABLE_ARN + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("ExportSummaries", hasSize(greaterThanOrEqualTo(1)))+ .body("ExportSummaries[0].ExportArn", notNullValue())+ .body("ExportSummaries[0].ExportStatus", notNullValue())+ .body("ExportSummaries[0].ExportType", equalTo("FULL_EXPORT"));+ }++ @Test+ void exportTableToPointInTime_unsupportedExportType_returnsValidationException() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableArn": "%s",+ "S3Bucket": "%s",+ "ExportType": "INCREMENTAL_EXPORT"+ }+ """.formatted(TABLE_ARN, BUCKET_NAME))+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("ValidationException"));+ }++ @Test+ void exportTableToPointInTime_ionFormat_returnsValidationException() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableArn": "%s",+ "S3Bucket": "%s",+ "ExportFormat": "ION"+ }+ """.formatted(TABLE_ARN, BUCKET_NAME))+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("ValidationException"));+ }++ @Test+ void exportTableToPointInTime_tableNotFound_returnsResourceNotFoundException() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableArn": "arn:aws:dynamodb:us-east-1:000000000000:table/NonExistentTable",+ "S3Bucket": "%s"+ }+ """.formatted(BUCKET_NAME))+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("ResourceNotFoundException"));+ }++ @Test+ void describeExport_notFound_returnsExportNotFoundException() {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"ExportArn\": \"arn:aws:dynamodb:us-east-1:000000000000:table/T/export/doesnotexist\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", containsString("ExportNotFoundException"));+ }++ @Test+ void exportData_s3ObjectsExist_andNdjsonIsValid() throws Exception {+ String exportArn = startExport("exports-data");+ pollUntilCompleted(exportArn, 10_000);++ String manifestKey = given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"ExportArn\": \"" + exportArn + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("ExportDescription.ExportManifest");++ assertNotNull(manifestKey, "ExportManifest key should be set");++ // Extract bucket prefix from the manifest key+ // manifestKey is like: exports-data/AWSDynamoDB/<exportId>/manifest-summary.json+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);+ String manifestFilesKey = "exports-data/AWSDynamoDB/" + exportId + "/manifest-files.json";++ // Download manifest-files.json and find the data key+ byte[] manifestFiles = given()+ .when().get("/" + BUCKET_NAME + "/" + manifestFilesKey)+ .then().statusCode(200)+ .extract().asByteArray();++ String dataKey = new String(manifestFiles, StandardCharsets.UTF_8).trim();+ assertFalse(dataKey.isEmpty(), "manifest-files.json should contain the data file key");+ assertTrue(dataKey.endsWith(".json.gz"), "Data file should be a .json.gz file");++ // Download and decompress the data file+ byte[] gzipData = given()+ .when().get("/" + BUCKET_NAME + "/" + dataKey)+ .then().statusCode(200)+ .extract().asByteArray();++ String ndjson = decompressGzip(gzipData);+ String[] lines = ndjson.split("\n");+ assertEquals(3, lines.length, "Should have 3 items in the export");++ for (String line : lines) {+ assertTrue(line.contains("\"Item\""), "Each line should have Item wrapper");+ assertTrue(line.contains("\"pk\""), "Each line should have pk attribute");+ }+ }++ // --- Helpers ---++ private void putItem(String itemJson) {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableName\": \"" + TABLE_NAME + "\", \"Item\": " + itemJson + "}")+ .when().post("/").then().statusCode(200);+ }++ private String startExport(String prefix) {+ return given()+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableArn": "%s",+ "S3Bucket": "%s",+ "S3Prefix": "%s"+ }+ """.formatted(TABLE_ARN, BUCKET_NAME, prefix))+ .when().post("/")+ .then().statusCode(200)+ .extract().path("ExportDescription.ExportArn");+ }++ private String pollUntilCompleted(String exportArn, long timeoutMs) {+ long deadline = System.currentTimeMillis() + timeoutMs;+ while (System.currentTimeMillis() < deadline) {+ String status = given()+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"ExportArn\": \"" + exportArn + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("ExportDescription.ExportStatus");++ if ("COMPLETED".equals(status) || "FAILED".equals(status)) {+ return status;+ }+ try {+ Thread.sleep(100);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return status;+ }+ }+ return "TIMEOUT";+ }++ private String decompressGzip(byte[] data) throws Exception {+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));+ BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {+ StringBuilder sb = new StringBuilder();+ String line;+ while ((line = reader.readLine()) != null) {+ if (!sb.isEmpty()) sb.append('\n');+ sb.append(line);+ }+ return sb.toString();+ }+ }+}