Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): implement ExportTableToPointInTime, DescribeExport, and ListExports (#653)

Hector Venturac91b74ae4ea9
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
new file mode 100644
index 0000000000..bdc1704dfd
--- /dev/null
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
@@ -0,0 +1,239 @@
+package com.floci.test;
+
+import org.junit.jupiter.api.*;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
+import software.amazon.awssdk.services.dynamodb.model.*;
+import software.amazon.awssdk.services.dynamodb.waiters.DynamoDbWaiter;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.*;
+
+import java.io.ByteArrayInputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+import static org.assertj.core.api.Assertions.*;
+
+@DisplayName("DynamoDB Export to S3")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbExportTest {
+
+ private static DynamoDbClient ddb;
+ private static S3Client s3;
+ private static final String TABLE_NAME = "sdk-export-test-table";
+ private static final String BUCKET_NAME = "sdk-export-test-bucket";
+ private static String tableArn;
+ private static String exportArn;
+
+ @BeforeAll
+ static void setup() {
+ ddb = TestFixtures.dynamoDbClient();
+ s3 = TestFixtures.s3Client();
+
+ // Create S3 bucket
+ try {
+ s3.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build());
+ } catch (BucketAlreadyExistsException | BucketAlreadyOwnedByYouException ignored) {}
+
+ // Create DynamoDB table
+ CreateTableResponse tableResp = ddb.createTable(CreateTableRequest.builder()
+ .tableName(TABLE_NAME)
+ .keySchema(
+ KeySchemaElement.builder().attributeName("pk").keyType(KeyType.HASH).build(),
+ KeySchemaElement.builder().attributeName("sk").keyType(KeyType.RANGE).build())
+ .attributeDefinitions(
+ AttributeDefinition.builder().attributeName("pk")
+ .attributeType(ScalarAttributeType.S).build(),
+ AttributeDefinition.builder().attributeName("sk")
+ .attributeType(ScalarAttributeType.S).build())
+ .billingMode(BillingMode.PAY_PER_REQUEST)
+ .build());
+ tableArn = tableResp.tableDescription().tableArn();
+
+ // Insert 3 items
+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
+ .item(Map.of(
+ "pk", AttributeValue.fromS("user-1"),
+ "sk", AttributeValue.fromS("order-001"),
+ "total", AttributeValue.fromN("99")))
+ .build());
+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
+ .item(Map.of(
+ "pk", AttributeValue.fromS("user-2"),
+ "sk", AttributeValue.fromS("order-002"),
+ "total", AttributeValue.fromN("55")))
+ .build());
+ ddb.putItem(PutItemRequest.builder().tableName(TABLE_NAME)
+ .item(Map.of(
+ "pk", AttributeValue.fromS("user-3"),
+ "sk", AttributeValue.fromS("order-003"),
+ "total", AttributeValue.fromN("150")))
+ .build());
+ }
+
+ @AfterAll
+ static void cleanup() {
+ try {
+ ddb.deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build());
+ } catch (Exception ignored) {}
+ try {
+ ListObjectsV2Response objects = s3.listObjectsV2(
+ ListObjectsV2Request.builder().bucket(BUCKET_NAME).build());
+ for (S3Object obj : objects.contents()) {
+ s3.deleteObject(DeleteObjectRequest.builder()
+ .bucket(BUCKET_NAME).key(obj.key()).build());
+ }
+ s3.deleteBucket(DeleteBucketRequest.builder().bucket(BUCKET_NAME).build());
+ } catch (Exception ignored) {}
+ if (ddb != null) ddb.close();
+ if (s3 != null) s3.close();
+ }
+
+ @Test
+ @Order(1)
+ void exportTableToPointInTime_returnsInProgressOrCompleted() {
+ ExportTableToPointInTimeResponse resp = ddb.exportTableToPointInTime(
+ ExportTableToPointInTimeRequest.builder()
+ .tableArn(tableArn)
+ .s3Bucket(BUCKET_NAME)
+ .s3Prefix("exports")
+ .exportFormat(ExportFormat.DYNAMODB_JSON)
+ .build());
+
+ ExportDescription desc = resp.exportDescription();
+ assertThat(desc.exportArn()).isNotBlank();
+ assertThat(desc.exportStatus()).isIn(ExportStatus.IN_PROGRESS, ExportStatus.COMPLETED);
+ assertThat(desc.tableArn()).isEqualTo(tableArn);
+ assertThat(desc.s3Bucket()).isEqualTo(BUCKET_NAME);
+ assertThat(desc.exportFormat()).isEqualTo(ExportFormat.DYNAMODB_JSON);
+ assertThat(desc.exportType()).isEqualTo(ExportType.FULL_EXPORT);
+
+ exportArn = desc.exportArn();
+ }
+
+ @Test
+ @Order(2)
+ void waitUntilExportCompleted_completesSuccessfully() {
+ assertThat(exportArn).isNotNull();
+
+ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {
+ waiter.waitUntilExportCompleted(r -> r.exportArn(exportArn));
+ }
+ }
+
+ @Test
+ @Order(3)
+ void describeExport_returnsCompletedExportWithAllFields() {
+ assertThat(exportArn).isNotNull();
+
+ DescribeExportResponse resp = ddb.describeExport(
+ DescribeExportRequest.builder().exportArn(exportArn).build());
+
+ ExportDescription desc = resp.exportDescription();
+ assertThat(desc.exportStatus()).isEqualTo(ExportStatus.COMPLETED);
+ assertThat(desc.tableArn()).isEqualTo(tableArn);
+ assertThat(desc.s3Bucket()).isEqualTo(BUCKET_NAME);
+ assertThat(desc.itemCount()).isEqualTo(3L);
+ assertThat(desc.billedSizeBytes()).isGreaterThan(0L);
+ assertThat(desc.exportManifest()).isNotBlank();
+ assertThat(desc.startTime()).isNotNull();
+ assertThat(desc.endTime()).isNotNull();
+ }
+
+ @Test
+ @Order(4)
+ void listExports_byTableArn_returnsExport() {
+ assertThat(exportArn).isNotNull();
+
+ ListExportsResponse resp = ddb.listExports(
+ ListExportsRequest.builder().tableArn(tableArn).build());
+
+ assertThat(resp.exportSummaries()).isNotEmpty();
+ assertThat(resp.exportSummaries().stream()
+ .anyMatch(s -> exportArn.equals(s.exportArn())))
+ .isTrue();
+
+ ExportSummary summary = resp.exportSummaries().stream()
+ .filter(s -> exportArn.equals(s.exportArn()))
+ .findFirst().orElseThrow();
+ assertThat(summary.exportStatus()).isEqualTo(ExportStatus.COMPLETED);
+ assertThat(summary.exportType()).isEqualTo(ExportType.FULL_EXPORT);
+ }
+
+ @Test
+ @Order(5)
+ void s3Objects_manifestAndDataExist() throws Exception {
+ assertThat(exportArn).isNotNull();
+
+ DescribeExportResponse descResp = ddb.describeExport(
+ DescribeExportRequest.builder().exportArn(exportArn).build());
+ String manifestSummaryKey = descResp.exportDescription().exportManifest();
+ assertThat(manifestSummaryKey).isNotBlank();
+
+ // Verify manifest-summary.json exists
+ ResponseBytes<GetObjectResponse> manifestSummary = s3.getObjectAsBytes(
+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestSummaryKey).build());
+ assertThat(manifestSummary.asByteArray().length).isGreaterThan(0);
+
+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
+ String manifestFilesKey = "exports/AWSDynamoDB/" + exportId + "/manifest-files.json";
+
+ ResponseBytes<GetObjectResponse> manifestFiles = s3.getObjectAsBytes(
+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestFilesKey).build());
+ String dataKey = new String(manifestFiles.asByteArray(), StandardCharsets.UTF_8).trim();
+ assertThat(dataKey).endsWith(".json.gz");
+
+ // Download and decompress the data file
+ ResponseBytes<GetObjectResponse> dataFile = s3.getObjectAsBytes(
+ GetObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());
+
+ String ndjson = decompressGzip(dataFile.asByteArray());
+ String[] lines = ndjson.split("\n");
+ assertThat(lines).hasSize(3);
+
+ for (String line : lines) {
+ assertThat(line).contains("\"Item\"");
+ assertThat(line).contains("\"pk\"");
+ }
+ }
+
+ @Test
+ @Order(6)
+ void exportTableToPointInTime_invalidExportType_throwsValidationException() {
+ assertThatThrownBy(() -> ddb.exportTableToPointInTime(
+ ExportTableToPointInTimeRequest.builder()
+ .tableArn(tableArn)
+ .s3Bucket(BUCKET_NAME)
+ .exportType(ExportType.INCREMENTAL_EXPORT)
+ .build()))
+ .isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class)
+ .hasMessageContaining("not supported");
+ }
+
+ @Test
+ @Order(7)
+ void describeExport_notFound_throwsExportNotFoundException() {
+ assertThatThrownBy(() -> ddb.describeExport(
+ DescribeExportRequest.builder()
+ .exportArn("arn:aws:dynamodb:us-east-1:000000000000:table/T/export/doesnotexist")
+ .build()))
+ .isInstanceOf(software.amazon.awssdk.services.dynamodb.model.DynamoDbException.class);
+ }
+
+ private String decompressGzip(byte[] data) throws Exception {
+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
+ StringBuilder sb = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (sb.length() > 0) sb.append('\n');
+ sb.append(line);
+ }
+ return sb.toString();
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index 155ea3035c..704bdaa75c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -63,6 +63,9 @@ public class DynamoDbJsonHandler {
case "EnableKinesisStreamingDestination" -> handleEnableKinesisStreamingDestination(request, region);
case "DisableKinesisStreamingDestination" -> handleDisableKinesisStreamingDestination(request, region);
case "DescribeKinesisStreamingDestination" -> handleDescribeKinesisStreamingDestination(request, region);
+ case "ExportTableToPointInTime" -> handleExportTable(request, region);
+ case "DescribeExport" -> handleDescribeExport(request, region);
+ case "ListExports" -> handleListExports(request, region);
default -> Response.status(400)
.entity(new AwsErrorResponse("UnknownOperationException", "Operation " + action + " is not supported."))
.build();
@@ -1005,4 +1008,47 @@ public class DynamoDbJsonHandler {
node.set("PointInTimeRecoveryDescription", pitrNode);
return node;
}
+
+ private Response handleExportTable(JsonNode request, String region) {
+ Map<String, Object> params = new java.util.HashMap<>();
+ request.fields().forEachRemaining(e -> params.put(e.getKey(), e.getValue().isTextual()
+ ? e.getValue().asText() : e.getValue()));
+
+ io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =
+ dynamoDbService.exportTable(params, region);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("ExportDescription", objectMapper.valueToTree(desc));
+ return Response.ok(response).build();
+ }
+
+ private Response handleDescribeExport(JsonNode request, String region) {
+ String exportArn = request.path("ExportArn").asText();
+ io.github.hectorvent.floci.services.dynamodb.model.ExportDescription desc =
+ dynamoDbService.describeExport(exportArn);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ response.set("ExportDescription", objectMapper.valueToTree(desc));
+ return Response.ok(response).build();
+ }
+
+ private Response handleListExports(JsonNode request, String region) {
+ String tableArn = request.has("TableArn") ? request.get("TableArn").asText() : null;
+ Integer maxResults = request.has("MaxResults") ? request.get("MaxResults").asInt() : null;
+ String nextToken = request.has("NextToken") && !request.get("NextToken").isNull()
+ ? request.get("NextToken").asText() : null;
+
+ DynamoDbService.ListExportsResult result = dynamoDbService.listExports(tableArn, maxResults, nextToken);
+
+ ObjectNode response = objectMapper.createObjectNode();
+ ArrayNode summaries = objectMapper.createArrayNode();
+ for (io.github.hectorvent.floci.services.dynamodb.model.ExportSummary s : result.exportSummaries()) {
+ summaries.add(objectMapper.valueToTree(s));
+ }
+ response.set("ExportSummaries", summaries);
+ if (result.nextToken() != null) {
+ response.put("NextToken", result.nextToken());
+ }
+ return Response.ok(response).build();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index c5ca5292b6..2ddbd3f931 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -5,31 +5,44 @@ import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.ExportDescription;
+import io.github.hectorvent.floci.services.dynamodb.model.ExportSummary;
import io.github.hectorvent.floci.services.dynamodb.model.GlobalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.LocalSecondaryIndex;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HexFormat;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;
+import java.util.zip.GZIPOutputStream;
@ApplicationScoped
public class DynamoDbService {
@@ -38,6 +51,7 @@ public class DynamoDbService {
private final StorageBackend<String, TableDefinition> tableStore;
private final StorageBackend<String, Map<String, JsonNode>> itemStore;
+ private final StorageBackend<String, ExportDescription> exportStore;
// Items stored per table: storageKey -> Map<itemKey, item>
// itemKey is "pk" or "pk#sk" depending on table schema
private final ConcurrentHashMap<String, ConcurrentSkipListMap<String, JsonNode>> itemsByTable = new ConcurrentHashMap<>();
@@ -47,33 +61,39 @@ public class DynamoDbService {
// not deadlock after the outer transaction already took each participant's lock.
private final ConcurrentHashMap<String, ConcurrentHashMap<String, ReentrantLock>> itemLocks = new ConcurrentHashMap<>();
private final RegionResolver regionResolver;
+ private final ObjectMapper objectMapper;
private DynamoDbStreamService streamService;
private KinesisStreamingForwarder kinesisForwarder;
+ private S3Service s3Service;
@Inject
public DynamoDbService(StorageFactory storageFactory, RegionResolver regionResolver,
DynamoDbStreamService streamService,
- KinesisStreamingForwarder kinesisForwarder) {
+ KinesisStreamingForwarder kinesisForwarder,
+ S3Service s3Service,
+ ObjectMapper objectMapper) {
this(storageFactory.create("dynamodb", "dynamodb-tables.json",
new TypeReference<Map<String, TableDefinition>>() {}),
storageFactory.create("dynamodb", "dynamodb-items.json",
new TypeReference<Map<String, Map<String, JsonNode>>>() {}),
- regionResolver, streamService, kinesisForwarder);
+ storageFactory.create("dynamodb", "dynamodb-exports.json",
+ new TypeReference<Map<String, ExportDescription>>() {}),
+ regionResolver, streamService, kinesisForwarder, s3Service, objectMapper);
}
/** Package-private constructor for testing. */
DynamoDbService(StorageBackend<String, TableDefinition> tableStore) {
- this(tableStore, null, new RegionResolver("us-east-1", "000000000000"), null, null);
+ this(tableStore, null, null, new RegionResolver("us-east-1", "000000000000"), null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore, RegionResolver regionResolver) {
- this(tableStore, null, regionResolver, null, null);
+ this(tableStore, null, null, regionResolver, null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
StorageBackend<String, Map<String, JsonNode>> itemStore,
RegionResolver regionResolver) {
- this(tableStore, itemStore, regionResolver, null, null);
+ this(tableStore, itemStore, null, regionResolver, null, null, null, null);
}
DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
@@ -81,11 +101,25 @@ public class DynamoDbService {
RegionResolver regionResolver,
DynamoDbStreamService streamService,
KinesisStreamingForwarder kinesisForwarder) {
+ this(tableStore, itemStore, null, regionResolver, streamService, kinesisForwarder, null, null);
+ }
+
+ DynamoDbService(StorageBackend<String, TableDefinition> tableStore,
+ StorageBackend<String, Map<String, JsonNode>> itemStore,
+ StorageBackend<String, ExportDescription> exportStore,
+ RegionResolver regionResolver,
+ DynamoDbStreamService streamService,
+ KinesisStreamingForwarder kinesisForwarder,
+ S3Service s3Service,
+ ObjectMapper objectMapper) {
this.tableStore = tableStore;
this.itemStore = itemStore;
+ this.exportStore = exportStore;
this.regionResolver = regionResolver;
this.streamService = streamService;
this.kinesisForwarder = kinesisForwarder;
+ this.s3Service = s3Service;
+ this.objectMapper = objectMapper != null ? objectMapper : new ObjectMapper();
loadPersistedItems();
}
@@ -1926,4 +1960,246 @@ public class DynamoDbService {
public record UpdateResult(JsonNode newItem, JsonNode oldItem) {}
public record ScanResult(List<JsonNode> items, int scannedCount, JsonNode lastEvaluatedKey) {}
public record QueryResult(List<JsonNode> items, int scannedCount, JsonNode lastEvaluatedKey) {}
+
+ // --- Export Operations ---
+
+ public ExportDescription exportTable(Map<String, Object> request, String region) {
+ String tableArn = (String) request.get("TableArn");
+ String s3Bucket = (String) request.get("S3Bucket");
+ String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;
+ String exportFormat = request.containsKey("ExportFormat") ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";
+ String exportType = request.containsKey("ExportType") ? (String) request.get("ExportType") : "FULL_EXPORT";
+ String clientToken = request.containsKey("ClientToken") ? (String) request.get("ClientToken") : null;
+ String s3SseAlgorithm = request.containsKey("S3SseAlgorithm") ? (String) request.get("S3SseAlgorithm") : null;
+ String s3BucketOwner = request.containsKey("S3BucketOwner") ? (String) request.get("S3BucketOwner") : null;
+
+ if ("INCREMENTAL_EXPORT".equals(exportType)) {
+ throw new AwsException("ValidationException",
+ "ExportType INCREMENTAL_EXPORT is not supported", 400);
+ }
+ if ("ION".equals(exportFormat)) {
+ throw new AwsException("ValidationException",
+ "ExportFormat ION is not supported", 400);
+ }
+
+ DynamoDbTableNames.ResolvedTableRef ref = DynamoDbTableNames.resolveWithRegion(tableArn, region);
+ String tableName = ref.name();
+ String tableRegion = ref.region() != null ? ref.region() : region;
+ String storageKey = regionKey(tableRegion, tableName);
+
+ TableDefinition table = tableStore.get(storageKey)
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",
+ "Requested resource not found: Table: " + tableName + " not found", 400));
+
+ long now = Instant.now().getEpochSecond();
+ String exportId = System.currentTimeMillis() + "-" + UUID.randomUUID().toString().replace("-", "");
+ String exportArn = "arn:aws:dynamodb:" + tableRegion + ":" + regionResolver.getAccountId()
+ + ":table/" + table.getTableName() + "/export/" + exportId;
+
+ ExportDescription desc = new ExportDescription();
+ desc.setExportArn(exportArn);
+ desc.setExportStatus("IN_PROGRESS");
+ desc.setTableArn(table.getTableArn());
+ desc.setTableId(table.getTableName());
+ desc.setS3Bucket(s3Bucket);
+ desc.setS3Prefix(s3Prefix);
+ desc.setExportFormat(exportFormat);
+ desc.setExportType("FULL_EXPORT");
+ desc.setExportTime(now);
+ desc.setStartTime(now);
+ desc.setClientToken(clientToken);
+ desc.setS3SseAlgorithm(s3SseAlgorithm);
+ desc.setS3BucketOwner(s3BucketOwner);
+
+ if (exportStore != null) {
+ exportStore.put(exportArn, desc);
+ }
+
+ ExportDescription finalDesc = desc;
+ ConcurrentSkipListMap<String, JsonNode> tableItems = itemsByTable.get(storageKey);
+ List<JsonNode> snapshot = tableItems != null
+ ? List.copyOf(tableItems.values())
+ : List.of();
+
+ Thread.ofVirtual().start(() -> runExport(finalDesc, snapshot, exportArn));
+
+ return desc;
+ }
+
+ private void runExport(ExportDescription desc, List<JsonNode> snapshot, String exportArn) {
+ try {
+ String s3Bucket = desc.getS3Bucket();
+ String s3Prefix = desc.getS3Prefix() != null ? desc.getS3Prefix() : "";
+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
+ String dataFileUuid = UUID.randomUUID().toString();
+ String dataKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")
+ + "AWSDynamoDB/" + exportId + "/data/" + dataFileUuid + ".json.gz";
+ String manifestFilesKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")
+ + "AWSDynamoDB/" + exportId + "/manifest-files.json";
+ String manifestSummaryKey = (s3Prefix.isEmpty() ? "" : s3Prefix + "/")
+ + "AWSDynamoDB/" + exportId + "/manifest-summary.json";
+
+ byte[] gzipData = buildGzipNdjson(snapshot);
+
+ try {
+ s3Service.putObject(s3Bucket, dataKey, gzipData, "application/octet-stream", Map.of());
+ } catch (AwsException e) {
+ if ("NoSuchBucket".equals(e.getErrorCode())) {
+ desc.setExportStatus("FAILED");
+ desc.setFailureCode("S3NoSuchBucket");
+ desc.setFailureMessage("The specified bucket does not exist: " + s3Bucket);
+ desc.setEndTime(Instant.now().getEpochSecond());
+ if (exportStore != null) {
+ exportStore.put(exportArn, desc);
+ }
+ return;
+ }
+ throw e;
+ }
+
+ String md5 = computeMd5Hex(gzipData);
+ String etag = md5;
+
+ String manifestFilesContent = dataKey + "\n";
+ s3Service.putObject(s3Bucket, manifestFilesKey,
+ manifestFilesContent.getBytes(StandardCharsets.UTF_8),
+ "application/json", Map.of());
+
+ long billedSize = gzipData.length;
+ long itemCount = snapshot.size();
+
+ String manifestSummaryContent = buildManifestSummary(
+ desc, exportId, dataKey, itemCount, billedSize, md5, etag, manifestSummaryKey);
+ s3Service.putObject(s3Bucket, manifestSummaryKey,
+ manifestSummaryContent.getBytes(StandardCharsets.UTF_8),
+ "application/json", Map.of());
+
+ long endTime = Instant.now().getEpochSecond();
+ desc.setExportStatus("COMPLETED");
+ desc.setEndTime(endTime);
+ desc.setItemCount(itemCount);
+ desc.setBilledSizeBytes(billedSize);
+ desc.setExportManifest(manifestSummaryKey);
+
+ if (exportStore != null) {
+ exportStore.put(exportArn, desc);
+ }
+ LOG.infov("Export completed: {0}, items={1}", exportArn, itemCount);
+
+ } catch (Exception e) {
+ LOG.errorv(e, "Export failed: {0}", exportArn);
+ desc.setExportStatus("FAILED");
+ desc.setFailureCode("UNKNOWN");
+ desc.setFailureMessage(e.getMessage());
+ desc.setEndTime(Instant.now().getEpochSecond());
+ if (exportStore != null) {
+ exportStore.put(exportArn, desc);
+ }
+ }
+ }
+
+ private byte[] buildGzipNdjson(List<JsonNode> items) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
+ for (JsonNode item : items) {
+ ObjectNode line = objectMapper.createObjectNode();
+ line.set("Item", item);
+ byte[] lineBytes = objectMapper.writeValueAsBytes(line);
+ gzip.write(lineBytes);
+ gzip.write('\n');
+ }
+ }
+ return baos.toByteArray();
+ }
+
+ private String computeMd5Hex(byte[] data) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ byte[] digest = md.digest(data);
+ return HexFormat.of().formatHex(digest);
+ } catch (NoSuchAlgorithmException e) {
+ return "unknown";
+ }
+ }
+
+ private String buildManifestSummary(ExportDescription desc, String exportId,
+ String dataKey, long itemCount, long billedSize,
+ String md5, String etag, String manifestSummaryKey) {
+ try {
+ com.fasterxml.jackson.databind.node.ObjectNode root = objectMapper.createObjectNode();
+ root.put("version", "2020-06-30");
+ root.put("exportArn", desc.getExportArn());
+ root.put("startTime", Instant.ofEpochSecond(desc.getStartTime()).toString());
+ root.put("endTime", Instant.now().toString());
+ root.put("tableArn", desc.getTableArn());
+ root.put("tableId", desc.getTableId());
+ root.put("exportTime", Instant.ofEpochSecond(desc.getExportTime()).toString());
+ root.put("s3Bucket", desc.getS3Bucket());
+ root.putNull("s3Prefix");
+ if (desc.getS3Prefix() != null) {
+ root.put("s3Prefix", desc.getS3Prefix());
+ }
+ root.put("s3SseAlgorithm", desc.getS3SseAlgorithm() != null ? desc.getS3SseAlgorithm() : "AES256");
+ root.putNull("s3SseKmsKeyId");
+ root.put("exportFormat", desc.getExportFormat());
+ root.put("billedSizeBytes", billedSize);
+ root.put("itemCount", itemCount);
+
+ com.fasterxml.jackson.databind.node.ArrayNode outputFiles = root.putArray("outputFiles");
+ com.fasterxml.jackson.databind.node.ObjectNode fileEntry = outputFiles.addObject();
+ fileEntry.put("itemCount", itemCount);
+ fileEntry.put("md5Checksum", md5);
+ fileEntry.put("etag", etag);
+ fileEntry.put("dataFileS3Key", dataKey);
+
+ return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(root);
+ } catch (JsonProcessingException e) {
+ return "{}";
+ }
+ }
+
+ public ExportDescription describeExport(String exportArn) {
+ if (exportStore == null) {
+ throw new AwsException("ExportNotFoundException",
+ "Export not found: " + exportArn, 400);
+ }
+ return exportStore.get(exportArn)
+ .orElseThrow(() -> new AwsException("ExportNotFoundException",
+ "Export not found: " + exportArn, 400));
+ }
+
+ public record ListExportsResult(List<ExportSummary> exportSummaries, String nextToken) {}
+
+ public ListExportsResult listExports(String tableArn, Integer maxResults, String nextToken) {
+ if (exportStore == null) {
+ return new ListExportsResult(List.of(), null);
+ }
+ int limit = maxResults != null ? Math.min(maxResults, 25) : 25;
+
+ List<ExportDescription> all = exportStore.keys().stream()
+ .map(k -> exportStore.get(k).orElse(null))
+ .filter(d -> d != null)
+ .filter(d -> tableArn == null || tableArn.equals(d.getTableArn()))
+ .sorted(Comparator.comparing(ExportDescription::getExportArn).reversed())
+ .toList();
+
+ int startIdx = 0;
+ if (nextToken != null) {
+ for (int i = 0; i < all.size(); i++) {
+ if (all.get(i).getExportArn().equals(nextToken)) {
+ startIdx = i + 1;
+ break;
+ }
+ }
+ }
+
+ List<ExportDescription> page = all.subList(startIdx, Math.min(startIdx + limit, all.size()));
+ String newNextToken = (startIdx + limit < all.size()) ? all.get(startIdx + limit - 1).getExportArn() : null;
+
+ List<ExportSummary> summaries = page.stream()
+ .map(ExportSummary::new)
+ .toList();
+
+ return new ListExportsResult(summaries, newNextToken);
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java
new file mode 100644
index 0000000000..4ae924b18d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportDescription.java
@@ -0,0 +1,126 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ExportDescription {
+
+ @JsonProperty("ExportArn")
+ private String exportArn;
+
+ @JsonProperty("ExportStatus")
+ private String exportStatus;
+
+ @JsonProperty("TableArn")
+ private String tableArn;
+
+ @JsonProperty("TableId")
+ private String tableId;
+
+ @JsonProperty("S3Bucket")
+ private String s3Bucket;
+
+ @JsonProperty("S3Prefix")
+ private String s3Prefix;
+
+ @JsonProperty("ExportFormat")
+ private String exportFormat;
+
+ @JsonProperty("ExportType")
+ private String exportType;
+
+ @JsonProperty("ExportTime")
+ private Long exportTime;
+
+ @JsonProperty("StartTime")
+ private Long startTime;
+
+ @JsonProperty("EndTime")
+ private Long endTime;
+
+ @JsonProperty("ItemCount")
+ private Long itemCount;
+
+ @JsonProperty("BilledSizeBytes")
+ private Long billedSizeBytes;
+
+ @JsonProperty("ExportManifest")
+ private String exportManifest;
+
+ @JsonProperty("ClientToken")
+ private String clientToken;
+
+ @JsonProperty("S3SseAlgorithm")
+ private String s3SseAlgorithm;
+
+ @JsonProperty("S3BucketOwner")
+ private String s3BucketOwner;
+
+ @JsonProperty("FailureCode")
+ private String failureCode;
+
+ @JsonProperty("FailureMessage")
+ private String failureMessage;
+
+ public String getExportArn() { return exportArn; }
+ public void setExportArn(String exportArn) { this.exportArn = exportArn; }
+
+ public String getExportStatus() { return exportStatus; }
+ public void setExportStatus(String exportStatus) { this.exportStatus = exportStatus; }
+
+ public String getTableArn() { return tableArn; }
+ public void setTableArn(String tableArn) { this.tableArn = tableArn; }
+
+ public String getTableId() { return tableId; }
+ public void setTableId(String tableId) { this.tableId = tableId; }
+
+ public String getS3Bucket() { return s3Bucket; }
+ public void setS3Bucket(String s3Bucket) { this.s3Bucket = s3Bucket; }
+
+ public String getS3Prefix() { return s3Prefix; }
+ public void setS3Prefix(String s3Prefix) { this.s3Prefix = s3Prefix; }
+
+ public String getExportFormat() { return exportFormat; }
+ public void setExportFormat(String exportFormat) { this.exportFormat = exportFormat; }
+
+ public String getExportType() { return exportType; }
+ public void setExportType(String exportType) { this.exportType = exportType; }
+
+ public Long getExportTime() { return exportTime; }
+ public void setExportTime(Long exportTime) { this.exportTime = exportTime; }
+
+ public Long getStartTime() { return startTime; }
+ public void setStartTime(Long startTime) { this.startTime = startTime; }
+
+ public Long getEndTime() { return endTime; }
+ public void setEndTime(Long endTime) { this.endTime = endTime; }
+
+ public Long getItemCount() { return itemCount; }
+ public void setItemCount(Long itemCount) { this.itemCount = itemCount; }
+
+ public Long getBilledSizeBytes() { return billedSizeBytes; }
+ public void setBilledSizeBytes(Long billedSizeBytes) { this.billedSizeBytes = billedSizeBytes; }
+
+ public String getExportManifest() { return exportManifest; }
+ public void setExportManifest(String exportManifest) { this.exportManifest = exportManifest; }
+
+ public String getClientToken() { return clientToken; }
+ public void setClientToken(String clientToken) { this.clientToken = clientToken; }
+
+ public String getS3SseAlgorithm() { return s3SseAlgorithm; }
+ public void setS3SseAlgorithm(String s3SseAlgorithm) { this.s3SseAlgorithm = s3SseAlgorithm; }
+
+ public String getS3BucketOwner() { return s3BucketOwner; }
+ public void setS3BucketOwner(String s3BucketOwner) { this.s3BucketOwner = s3BucketOwner; }
+
+ public String getFailureCode() { return failureCode; }
+ public void setFailureCode(String failureCode) { this.failureCode = failureCode; }
+
+ public String getFailureMessage() { return failureMessage; }
+ public void setFailureMessage(String failureMessage) { this.failureMessage = failureMessage; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.java
new file mode 100644
index 0000000000..7eedbfbeaa
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/model/ExportSummary.java
@@ -0,0 +1,38 @@
+package io.github.hectorvent.floci.services.dynamodb.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+@RegisterForReflection
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ExportSummary {
+
+ @JsonProperty("ExportArn")
+ private String exportArn;
+
+ @JsonProperty("ExportStatus")
+ private String exportStatus;
+
+ @JsonProperty("ExportType")
+ private String exportType;
+
+ public ExportSummary() {}
+
+ public ExportSummary(ExportDescription desc) {
+ this.exportArn = desc.getExportArn();
+ this.exportStatus = desc.getExportStatus();
+ this.exportType = desc.getExportType();
+ }
+
+ public String getExportArn() { return exportArn; }
+ public void setExportArn(String exportArn) { this.exportArn = exportArn; }
+
+ public String getExportStatus() { return exportStatus; }
+ public void setExportStatus(String exportStatus) { this.exportStatus = exportStatus; }
+
+ public String getExportType() { return exportType; }
+ public void setExportType(String exportType) { this.exportType = exportType; }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
new file mode 100644
index 0000000000..0cbce5eb55
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
@@ -0,0 +1,315 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.zip.GZIPInputStream;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.*;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class DynamoDbExportIntegrationTest {
+
+ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String TABLE_NAME = "ExportTestTable";
+ private static final String BUCKET_NAME = "export-test-bucket";
+ private static final String TABLE_ARN =
+ "arn:aws:dynamodb:us-east-1:000000000000:table/" + TABLE_NAME;
+
+ private static boolean setupDone = false;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @BeforeEach
+ void ensureSetup() {
+ if (setupDone) return;
+
+ // Create S3 bucket
+ given()
+ .when().put("/" + BUCKET_NAME)
+ .then().statusCode(anyOf(equalTo(200), equalTo(409)));
+
+ // Create table
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "%s",
+ "KeySchema": [
+ {"AttributeName": "pk", "KeyType": "HASH"},
+ {"AttributeName": "sk", "KeyType": "RANGE"}
+ ],
+ "AttributeDefinitions": [
+ {"AttributeName": "pk", "AttributeType": "S"},
+ {"AttributeName": "sk", "AttributeType": "S"}
+ ]
+ }
+ """.formatted(TABLE_NAME))
+ .when().post("/").then().statusCode(200);
+
+ // Insert 3 items
+ putItem("{\"pk\": {\"S\": \"user-1\"}, \"sk\": {\"S\": \"order-001\"}, \"total\": {\"N\": \"99\"}}");
+ putItem("{\"pk\": {\"S\": \"user-2\"}, \"sk\": {\"S\": \"order-002\"}, \"total\": {\"N\": \"55\"}}");
+ putItem("{\"pk\": {\"S\": \"user-3\"}, \"sk\": {\"S\": \"order-003\"}, \"total\": {\"N\": \"150\"}}");
+
+ setupDone = true;
+ }
+
+ @Test
+ void exportTableToPointInTime_returnsInProgressOrCompleted() {
+ String exportArn = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableArn": "%s",
+ "S3Bucket": "%s",
+ "S3Prefix": "exports",
+ "ExportFormat": "DYNAMODB_JSON"
+ }
+ """.formatted(TABLE_ARN, BUCKET_NAME))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("ExportDescription.ExportArn", notNullValue())
+ .body("ExportDescription.ExportStatus", oneOf("IN_PROGRESS", "COMPLETED"))
+ .body("ExportDescription.TableArn", equalTo(TABLE_ARN))
+ .body("ExportDescription.S3Bucket", equalTo(BUCKET_NAME))
+ .body("ExportDescription.ExportFormat", equalTo("DYNAMODB_JSON"))
+ .body("ExportDescription.ExportType", equalTo("FULL_EXPORT"))
+ .extract().path("ExportDescription.ExportArn");
+
+ assertNotNull(exportArn);
+ assertTrue(exportArn.contains("/export/"), "ExportArn should contain /export/ segment");
+
+ // Poll DescribeExport until COMPLETED (max 10s)
+ String status = pollUntilCompleted(exportArn, 10_000);
+ assertEquals("COMPLETED", status, "Export should complete within 10 seconds");
+ }
+
+ @Test
+ void describeExport_returnsCompletedExportWithS3Manifest() throws Exception {
+ String exportArn = startExport("exports-describe");
+ String status = pollUntilCompleted(exportArn, 10_000);
+ assertEquals("COMPLETED", status);
+
+ // Verify ExportManifest is set
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"ExportArn\": \"" + exportArn + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("ExportDescription.ExportStatus", equalTo("COMPLETED"))
+ .body("ExportDescription.ItemCount", equalTo(3))
+ .body("ExportDescription.BilledSizeBytes", greaterThan(0))
+ .body("ExportDescription.ExportManifest", notNullValue());
+ }
+
+ @Test
+ void listExports_returnsCompletedExport() throws Exception {
+ String exportArn = startExport("exports-list");
+ pollUntilCompleted(exportArn, 10_000);
+
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ListExports")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableArn\": \"" + TABLE_ARN + "\"}")
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("ExportSummaries", hasSize(greaterThanOrEqualTo(1)))
+ .body("ExportSummaries[0].ExportArn", notNullValue())
+ .body("ExportSummaries[0].ExportStatus", notNullValue())
+ .body("ExportSummaries[0].ExportType", equalTo("FULL_EXPORT"));
+ }
+
+ @Test
+ void exportTableToPointInTime_unsupportedExportType_returnsValidationException() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableArn": "%s",
+ "S3Bucket": "%s",
+ "ExportType": "INCREMENTAL_EXPORT"
+ }
+ """.formatted(TABLE_ARN, BUCKET_NAME))
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"));
+ }
+
+ @Test
+ void exportTableToPointInTime_ionFormat_returnsValidationException() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableArn": "%s",
+ "S3Bucket": "%s",
+ "ExportFormat": "ION"
+ }
+ """.formatted(TABLE_ARN, BUCKET_NAME))
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ValidationException"));
+ }
+
+ @Test
+ void exportTableToPointInTime_tableNotFound_returnsResourceNotFoundException() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableArn": "arn:aws:dynamodb:us-east-1:000000000000:table/NonExistentTable",
+ "S3Bucket": "%s"
+ }
+ """.formatted(BUCKET_NAME))
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ResourceNotFoundException"));
+ }
+
+ @Test
+ void describeExport_notFound_returnsExportNotFoundException() {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"ExportArn\": \"arn:aws:dynamodb:us-east-1:000000000000:table/T/export/doesnotexist\"}")
+ .when().post("/")
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ExportNotFoundException"));
+ }
+
+ @Test
+ void exportData_s3ObjectsExist_andNdjsonIsValid() throws Exception {
+ String exportArn = startExport("exports-data");
+ pollUntilCompleted(exportArn, 10_000);
+
+ String manifestKey = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"ExportArn\": \"" + exportArn + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().path("ExportDescription.ExportManifest");
+
+ assertNotNull(manifestKey, "ExportManifest key should be set");
+
+ // Extract bucket prefix from the manifest key
+ // manifestKey is like: exports-data/AWSDynamoDB/<exportId>/manifest-summary.json
+ String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
+ String manifestFilesKey = "exports-data/AWSDynamoDB/" + exportId + "/manifest-files.json";
+
+ // Download manifest-files.json and find the data key
+ byte[] manifestFiles = given()
+ .when().get("/" + BUCKET_NAME + "/" + manifestFilesKey)
+ .then().statusCode(200)
+ .extract().asByteArray();
+
+ String dataKey = new String(manifestFiles, StandardCharsets.UTF_8).trim();
+ assertFalse(dataKey.isEmpty(), "manifest-files.json should contain the data file key");
+ assertTrue(dataKey.endsWith(".json.gz"), "Data file should be a .json.gz file");
+
+ // Download and decompress the data file
+ byte[] gzipData = given()
+ .when().get("/" + BUCKET_NAME + "/" + dataKey)
+ .then().statusCode(200)
+ .extract().asByteArray();
+
+ String ndjson = decompressGzip(gzipData);
+ String[] lines = ndjson.split("\n");
+ assertEquals(3, lines.length, "Should have 3 items in the export");
+
+ for (String line : lines) {
+ assertTrue(line.contains("\"Item\""), "Each line should have Item wrapper");
+ assertTrue(line.contains("\"pk\""), "Each line should have pk attribute");
+ }
+ }
+
+ // --- Helpers ---
+
+ private void putItem(String itemJson) {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"" + TABLE_NAME + "\", \"Item\": " + itemJson + "}")
+ .when().post("/").then().statusCode(200);
+ }
+
+ private String startExport(String prefix) {
+ return given()
+ .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableArn": "%s",
+ "S3Bucket": "%s",
+ "S3Prefix": "%s"
+ }
+ """.formatted(TABLE_ARN, BUCKET_NAME, prefix))
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().path("ExportDescription.ExportArn");
+ }
+
+ private String pollUntilCompleted(String exportArn, long timeoutMs) {
+ long deadline = System.currentTimeMillis() + timeoutMs;
+ while (System.currentTimeMillis() < deadline) {
+ String status = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DescribeExport")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"ExportArn\": \"" + exportArn + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().path("ExportDescription.ExportStatus");
+
+ if ("COMPLETED".equals(status) || "FAILED".equals(status)) {
+ return status;
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return status;
+ }
+ }
+ return "TIMEOUT";
+ }
+
+ private String decompressGzip(byte[] data) throws Exception {
+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
+ StringBuilder sb = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (!sb.isEmpty()) sb.append('\n');
+ sb.append(line);
+ }
+ return sb.toString();
+ }
+ }
+}