Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): export tables in the ION format (#6429)

TATSUNO “Taz” Yasuhiroed21e67cd46a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
index 3245c62929..5545cff16c 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java
@@ -3,7 +3,6 @@ package com.floci.test;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.*;
-import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.dynamodb.model.*;
import software.amazon.awssdk.services.dynamodb.waiters.DynamoDbWaiter;
@@ -178,34 +177,18 @@ class DynamoDbExportTest {
void s3Objects_manifestAndDataExist() throws Exception {
assertThat(exportArn).isNotNull();
- DescribeExportResponse descResp = ddb.describeExport(
- DescribeExportRequest.builder().exportArn(exportArn).build());
- String manifestSummaryKey = descResp.exportDescription().exportManifest();
- assertThat(manifestSummaryKey).isNotBlank();
-
- // Verify manifest-summary.json exists
- ResponseBytes<GetObjectResponse> manifestSummary = s3.getObjectAsBytes(
- GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestSummaryKey).build());
- assertThat(new ObjectMapper().readTree(manifestSummary.asByteArray()).get("outputFormat").asText())
- .isEqualTo("DYNAMODB_JSON");
-
+ ExportFiles files = readExportFiles(exportArn);
+ assertThat(files.manifestSummary().get("outputFormat").asText()).isEqualTo("DYNAMODB_JSON");
String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);
- String manifestFilesKey = "exports/AWSDynamoDB/" + exportId + "/manifest-files.json";
+ assertThat(files.manifestSummary().get("manifestFilesS3Key").asText())
+ .isEqualTo("exports/AWSDynamoDB/" + exportId + "/manifest-files.json");
- ResponseBytes<GetObjectResponse> manifestFiles = s3.getObjectAsBytes(
- GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestFilesKey).build());
- JsonNode dataFileEntry = new ObjectMapper().readTree(manifestFiles.asByteArray());
- String dataKey = dataFileEntry.get("dataFileS3Key").asText();
+ String dataKey = files.dataFileEntry().get("dataFileS3Key").asText();
assertThat(dataKey).endsWith(".json.gz");
HeadObjectResponse head = s3.headObject(HeadObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());
- assertThat(head.eTag()).isEqualTo("\"" + dataFileEntry.get("etag").asText() + "\"").endsWith("-1\"");
+ assertThat(head.eTag()).isEqualTo("\"" + files.dataFileEntry().get("etag").asText() + "\"").endsWith("-1\"");
- // Download and decompress the data file
- ResponseBytes<GetObjectResponse> dataFile = s3.getObjectAsBytes(
- GetObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());
-
- String ndjson = decompressGzip(dataFile.asByteArray());
- String[] lines = ndjson.split("\n");
+ String[] lines = files.data().split("\n");
assertThat(lines).hasSize(3);
for (String line : lines) {
@@ -277,6 +260,47 @@ class DynamoDbExportTest {
.hasMessageContaining("Export Time is invalid");
}
+ @Test
+ @Order(10)
+ void exportTableToPointInTime_ionFormat_writesIonLines() throws Exception {
+ ExportDescription desc = ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()
+ .tableArn(tableArn)
+ .s3Bucket(BUCKET_NAME)
+ .s3Prefix("exports-ion")
+ .exportFormat(ExportFormat.ION)
+ .build()).exportDescription();
+ assertThat(desc.exportFormat()).isEqualTo(ExportFormat.ION);
+
+ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {
+ waiter.waitUntilExportCompleted(r -> r.exportArn(desc.exportArn()));
+ }
+ ExportFiles files = readExportFiles(desc.exportArn());
+ assertThat(files.manifestSummary().get("outputFormat").asText()).isEqualTo("ION");
+ assertThat(files.dataFileEntry().get("dataFileS3Key").asText()).endsWith(".ion.gz");
+ assertThat(files.data().split("\n")).hasSize(4)
+ .allSatisfy(line -> assertThat(line).startsWith("$ion_1_0 {Item:{").endsWith("}}"))
+ .anySatisfy(line -> assertThat(line).contains("pk:\"user-3\"", "sk:\"order-003\"", "total:15d1"));
+ }
+
+ /** The manifests and the data of an export that wrote one data file. */
+ private record ExportFiles(JsonNode manifestSummary, JsonNode dataFileEntry, String data) {}
+
+ private ExportFiles readExportFiles(String arn) throws Exception {
+ JsonNode manifestSummary = readJson(ddb.describeExport(DescribeExportRequest.builder().exportArn(arn).build())
+ .exportDescription().exportManifest());
+ JsonNode dataFileEntry = readJson(manifestSummary.get("manifestFilesS3Key").asText());
+ String data = decompressGzip(getObject(dataFileEntry.get("dataFileS3Key").asText()));
+ return new ExportFiles(manifestSummary, dataFileEntry, data);
+ }
+
+ private JsonNode readJson(String key) throws Exception {
+ return new ObjectMapper().readTree(getObject(key));
+ }
+
+ private byte[] getObject(String key) {
+ return s3.getObjectAsBytes(GetObjectRequest.builder().bucket(BUCKET_NAME).key(key).build()).asByteArray();
+ }
+
private String decompressGzip(byte[] data) throws Exception {
try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));
BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index 9ae37f1a5d..0f27b2dd16 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -409,7 +409,7 @@ Deviations from AWS:
## Export to S3
-Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format). As on AWS, the table needs point-in-time recovery:
+Export table data to an S3 bucket as gzip-compressed lines of DynamoDB JSON (`DYNAMODB_JSON`, the default) or Amazon Ion text (`ION`). As on AWS, the table needs point-in-time recovery:
```bash
# Create a bucket to receive the export
@@ -440,7 +440,7 @@ aws dynamodb list-exports \
--endpoint-url $AWS_ENDPOINT_URL
```
-The export writes `_started`, `manifest-summary.json`, `manifest-files.json` and an `.md5` file for each manifest to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/`, and the items to a `.json.gz` file in its `data/` folder, the layout AWS uses. AWS writes one data file per partition. Floci writes one. As on AWS, every file is a multipart upload of one part, so its ETag ends in `-1`. As on AWS, an `S3Prefix` shorter than 3 characters or longer than 948 UTF-8 bytes is refused, and no slash is added after a prefix that ends with one.
+The export writes `_started`, `manifest-summary.json`, `manifest-files.json` and an `.md5` file for each manifest to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/`, and the items to a `.json.gz` or `.ion.gz` file in its `data/` folder, the layout AWS uses. AWS writes one data file per partition. Floci writes one. An Ion line is `$ion_1_0 {Item:{...}}`, with sets annotated as `$dynamodb_SS`, `$dynamodb_NS` and `$dynamodb_BS`, and numbers as Ion decimals. As on AWS, every file is a multipart upload of one part, so its ETag ends in `-1`. As on AWS, an `S3Prefix` shorter than 3 characters or longer than 948 UTF-8 bytes is refused, and no slash is added after a prefix that ends with one.
The export holds the items as they were at `ExportTime`, or at the request time when `ExportTime` is not set. As on AWS, `ExportTime` must not be before `EarliestRestorableDateTime` or the table creation, and must not be in the future. A time after `LatestRestorableDateTime` is accepted.
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java
new file mode 100644
index 0000000000..05161b50a1
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java
@@ -0,0 +1,140 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.math.BigDecimal;
+import java.util.HexFormat;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.regex.Pattern;
+
+/**
+ * Writes items as the Ion text of an ION export. AWS prints them with the standard text writer of
+ * ion-java, so this follows its quoting, escapes and decimal forms.
+ */
+final class DynamoDbIonText {
+
+ private static final Pattern IDENTIFIER = Pattern.compile("[A-Za-z_$][A-Za-z0-9_$]*");
+ /** Ion reads these as symbol ids, so they are quoted like the keywords. */
+ private static final Pattern SYMBOL_ID = Pattern.compile("\\$[0-9]+");
+ private static final Set<String> KEYWORDS = Set.of("null", "true", "false", "nan");
+
+ private DynamoDbIonText() {}
+
+ /** One line of a full export, without its newline. */
+ static String exportLine(JsonNode item) {
+ StringBuilder out = new StringBuilder("$ion_1_0 {Item:");
+ appendStruct(out, item);
+ return out.append('}').toString();
+ }
+
+ private static void appendStruct(StringBuilder out, JsonNode attributes) {
+ out.append('{');
+ String separator = "";
+ for (Map.Entry<String, JsonNode> attribute : attributes.properties()) {
+ out.append(separator);
+ separator = ",";
+ appendFieldName(out, attribute.getKey());
+ out.append(':');
+ appendValue(out, attribute.getValue());
+ }
+ out.append('}');
+ }
+
+ private static void appendValue(StringBuilder out, JsonNode value) {
+ Map.Entry<String, JsonNode> typed = value.properties().iterator().next();
+ JsonNode content = typed.getValue();
+ switch (typed.getKey()) {
+ case "S" -> appendString(out, content);
+ case "N" -> appendDecimal(out, content);
+ case "B" -> appendBlob(out, content);
+ case "BOOL" -> out.append(content.asBoolean());
+ case "NULL" -> out.append("null");
+ case "SS" -> appendList(out.append("$dynamodb_SS::"), content, DynamoDbIonText::appendString);
+ case "NS" -> appendList(out.append("$dynamodb_NS::"), content, DynamoDbIonText::appendDecimal);
+ case "BS" -> appendList(out.append("$dynamodb_BS::"), content, DynamoDbIonText::appendBlob);
+ case "L" -> appendList(out, content, DynamoDbIonText::appendValue);
+ case "M" -> appendStruct(out, content);
+ default -> throw new IllegalArgumentException("Unknown attribute type: " + typed.getKey());
+ }
+ }
+
+ private static void appendList(StringBuilder out, JsonNode members, BiConsumer<StringBuilder, JsonNode> appendMember) {
+ out.append('[');
+ String separator = "";
+ for (JsonNode member : members) {
+ out.append(separator);
+ separator = ",";
+ appendMember.accept(out, member);
+ }
+ out.append(']');
+ }
+
+ private static void appendFieldName(StringBuilder out, String name) {
+ if (IDENTIFIER.matcher(name).matches() && !KEYWORDS.contains(name) && !SYMBOL_ID.matcher(name).matches()) {
+ out.append(name);
+ } else {
+ out.append('\'');
+ appendEscaped(out, name, '\'');
+ out.append('\'');
+ }
+ }
+
+ private static void appendString(StringBuilder out, JsonNode text) {
+ out.append('"');
+ appendEscaped(out, text.asText(), '"');
+ out.append('"');
+ }
+
+ private static void appendEscaped(StringBuilder out, String text, char quote) {
+ for (int i = 0; i < text.length(); i++) {
+ char c = text.charAt(i);
+ switch (c) {
+ case 0 -> out.append("\\0");
+ case 7 -> out.append("\\a");
+ case '\b' -> out.append("\\b");
+ case '\t' -> out.append("\\t");
+ case '\n' -> out.append("\\n");
+ case 0x0B -> out.append("\\v");
+ case '\f' -> out.append("\\f");
+ case '\r' -> out.append("\\r");
+ case '\\' -> out.append("\\\\");
+ default -> {
+ if (c == quote) {
+ out.append('\\').append(c);
+ } else if (c < 0x20 || (0x7F <= c && c <= 0x9F)) {
+ out.append("\\x").append(HexFormat.of().toHexDigits((byte) c));
+ } else {
+ out.append(c);
+ }
+ }
+ }
+ }
+ }
+
+ private static void appendDecimal(StringBuilder out, JsonNode number) {
+ BigDecimal value = new BigDecimal(number.asText()).stripTrailingZeros();
+ if (value.signum() < 0) {
+ out.append('-');
+ }
+ String digits = value.unscaledValue().abs().toString();
+ int scale = value.scale();
+ if (scale < 0) {
+ out.append(digits).append('d').append(-scale);
+ } else if (scale == 0) {
+ out.append(digits).append('.');
+ } else if (scale < digits.length()) {
+ int whole = digits.length() - scale;
+ out.append(digits, 0, whole).append('.').append(digits, whole, digits.length());
+ } else if (scale <= digits.length() + 5) {
+ out.append("0.").append("0".repeat(scale - digits.length())).append(digits);
+ } else {
+ out.append(digits).append("d-").append(scale);
+ }
+ }
+
+ private static void appendBlob(StringBuilder out, JsonNode base64) {
+ out.append("{{").append(base64.asText()).append("}}");
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 2e3bd8aa76..d468b285a6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -4918,37 +4918,39 @@ public class DynamoDbService {
// --- Export Operations ---
+ private static final List<String> EXPORT_FORMATS = List.of("ION", "DYNAMODB_JSON");
+
/** As on AWS, an ExportTime after LatestRestorableDateTime is accepted up to now. */
public ExportDescription exportTable(Map<String, Object> request, Instant exportTime, String region) {
String tableArn = (String) request.get("TableArn");
String s3Bucket = (String) request.get("S3Bucket");
String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;
- String exportFormat = request.containsKey("ExportFormat") ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";
+ String exportFormat = request.get("ExportFormat") != null ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";
String exportType = request.containsKey("ExportType") ? (String) request.get("ExportType") : "FULL_EXPORT";
String clientToken = request.containsKey("ClientToken") ? (String) request.get("ClientToken") : null;
String s3SseAlgorithm = request.containsKey("S3SseAlgorithm") ? (String) request.get("S3SseAlgorithm") : null;
String s3BucketOwner = request.containsKey("S3BucketOwner") ? (String) request.get("S3BucketOwner") : null;
- if (s3Prefix != null) {
- int length = s3Prefix.codePointCount(0, s3Prefix.length());
- if (1024 < length) {
- throw SmithyValidationErrors.validationException("Value '" + s3Prefix
- + "' at 's3Prefix' failed to satisfy constraint: Member must have length less than or equal to 1024");
- }
- if (length < 3) {
- throw new AwsException("ValidationException", "Invalid Request: " + SmithyValidationErrors.message(
- List.of("Value '" + s3Prefix + "' at 's3Prefix' failed to satisfy constraint: "
- + "Member must have length greater than or equal to 3")), 400);
- }
+ List<String> violations = new ArrayList<>();
+ if (s3Prefix != null && 1024 < s3Prefix.codePointCount(0, s3Prefix.length())) {
+ violations.add("Value '" + s3Prefix
+ + "' at 's3Prefix' failed to satisfy constraint: Member must have length less than or equal to 1024");
+ }
+ if (!EXPORT_FORMATS.contains(exportFormat)) {
+ violations.add(SmithyValidationErrors.notInEnum(exportFormat, "exportFormat", EXPORT_FORMATS));
+ }
+ if (!violations.isEmpty()) {
+ throw SmithyValidationErrors.validationException(violations);
+ }
+ if (s3Prefix != null && s3Prefix.codePointCount(0, s3Prefix.length()) < 3) {
+ throw new AwsException("ValidationException", "Invalid Request: " + SmithyValidationErrors.message(
+ List.of("Value '" + s3Prefix + "' at 's3Prefix' failed to satisfy constraint: "
+ + "Member must have length greater than or equal to 3")), 400);
}
if ("INCREMENTAL_EXPORT".equals(exportType)) {
throw new AwsException("ValidationException",
"ExportType INCREMENTAL_EXPORT is not supported", 400);
}
- if ("ION".equals(exportFormat)) {
- throw new AwsException("ValidationException",
- "ExportFormat ION is not supported", 400);
- }
DynamoDbTableNames.ResolvedTableRef ref = DynamoDbTableNames.resolveWithRegion(tableArn, region);
String tableName = ref.name();
@@ -5037,8 +5039,9 @@ public class DynamoDbService {
throw e;
}
- byte[] gzipData = buildGzipNdjson(snapshot);
- String dataKey = exportDir + "data/" + exportFileName() + ".json.gz";
+ boolean ion = "ION".equals(desc.getExportFormat());
+ byte[] gzipData = buildGzipExportData(snapshot, ion);
+ String dataKey = exportDir + "data/" + exportFileName() + (ion ? ".ion.gz" : ".json.gz");
ObjectNode dataFile = objectMapper.createObjectNode();
dataFile.put("itemCount", snapshot.size());
dataFile.put("md5Checksum", Base64.getEncoder().encodeToString(Digests.md5(gzipData)));
@@ -5072,14 +5075,18 @@ public class DynamoDbService {
}
}
- private byte[] buildGzipNdjson(List<JsonNode> items) throws IOException {
+ private byte[] buildGzipExportData(List<JsonNode> items, boolean ion) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {
for (JsonNode item : items) {
- ObjectNode line = objectMapper.createObjectNode();
- line.set("Item", item);
- byte[] lineBytes = objectMapper.writeValueAsBytes(line);
- gzip.write(lineBytes);
+ if (ion) {
+ gzip.write(DynamoDbIonText.exportLine(item).getBytes(StandardCharsets.UTF_8));
+ } else {
+ ObjectNode line = objectMapper.createObjectNode();
+ line.set("Item", item);
+ byte[] lineBytes = objectMapper.writeValueAsBytes(line);
+ gzip.write(lineBytes);
+ }
gzip.write('\n');
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
index 1f7b0d95ac..6ec9aef238 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java
@@ -176,21 +176,23 @@ class DynamoDbExportIntegrationTest {
}
@Test
- void exportTableToPointInTime_ionFormat_returnsValidationException() {
- given()
- .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")
- .contentType(DYNAMODB_CONTENT_TYPE)
- .body("""
- {
- "TableArn": "%s",
- "S3Bucket": "%s",
- "ExportFormat": "ION"
- }
- """.formatted(TABLE_ARN, BUCKET_NAME))
- .when().post("/")
- .then()
- .statusCode(400)
- .body("__type", containsString("ValidationException"));
+ void exportFormatIsCheckedAsAwsChecksIt() {
+ String enumSet = "' at 'exportFormat' failed to satisfy constraint: "
+ + "Member must satisfy enum value set: [ION, DYNAMODB_JSON]";
+ expectError(ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "ExportFormat": "ion", "S3Prefix": "x"}
+ """.formatted(TABLE_ARN_PREFIX + "ExportFormatMissing", BUCKET_NAME)), "ValidationException",
+ "1 validation error detected: Value 'ion" + enumSet);
+ String over1024 = "a".repeat(1025);
+ expectError(ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "ExportFormat": "", "S3Prefix": "%s"}
+ """.formatted(TABLE_ARN, BUCKET_NAME, over1024)), "ValidationException",
+ "2 validation errors detected: Value '" + over1024 + "' at 's3Prefix' failed to satisfy constraint: "
+ + "Member must have length less than or equal to 1024; Value '" + enumSet);
+ ddb("ExportTableToPointInTime", """
+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "exports-ion", "ExportFormat": "ION"}
+ """.formatted(TABLE_ARN, BUCKET_NAME)).statusCode(200)
+ .body("ExportDescription.ExportFormat", equalTo("ION"));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.java
new file mode 100644
index 0000000000..90443c11b2
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.java
@@ -0,0 +1,124 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.amazon.ion.IonType;
+import com.amazon.ion.IonWriter;
+import com.amazon.ion.system.IonTextWriterBuilder;
+import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class DynamoDbIonTextTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ void exportLineIsTheLineAwsWrites() throws IOException {
+ JsonNode item = MAPPER.readTree("""
+ {"pk": {"S": "types"}, "sk": {"N": "1"}, "b": {"B": "AAEC/w=="}, "n_lead": {"N": "007"},
+ "ns": {"NS": ["2.50", "1E+3", "3", "1"]}, "null": {"S": "kw null"}, "true": {"S": "kw true"},
+ "s_unicode": {"S": "日本語 é 😀"},
+ "l": {"L": [{"S": "x"}, {"N": "1"}, {"BOOL": true}, {"NULL": true}, {"L": []}, {"M": {}},
+ {"M": {"a": {"N": "1"}}}, {"SS": ["z"]}]},
+ "s_empty": {"S": ""}, "m": {"M": {"a-b": {"N": "1"}, "nested": {"M": {"deeper": {"S": "v"}}}}},
+ "bool_t": {"BOOL": true}, "n_int": {"N": "103"}, "bs": {"BS": ["AA==", "eA=="]}, "s": {"S": "hello"},
+ "a-b": {"S": "dash"}, "$x": {"S": "dollar"}, "n_zero": {"N": "0"},
+ "n_38": {"N": "12345678901234567890123456789012345678"}, "n_trail": {"N": "123.4500"},
+ "s_quote": {"S": "he said \\"hi\\"\\n\\ttab \\\\ back 'single'"}, "n_point": {"N": "10.0"},
+ "n_2000": {"N": "2000"}, "m_empty": {"M": {}}, "ss": {"SS": ["a", "b", "c d"]},
+ "n_big": {"N": "1E+125"}, "bool_f": {"BOOL": false}, "$10": {"S": "dollar digits"},
+ "with space": {"S": "sp"}, "n_frac": {"N": "0.5"}, "nul": {"NULL": true}, "n_small": {"N": "1E-130"},
+ "n_600": {"N": "600"}, "'q": {"S": "quote"}, "b_empty": {"B": ""}, "n_negfrac": {"N": "-0.001"},
+ "n_neg": {"N": "-1"}, "n_e": {"N": "1.5e3"}, "+inf": {"S": "inf"},
+ "s_ctrl": {"S": "\\u0001\\u001f\\u007f\\u0080"}, "a.b": {"N": "2"}, "日本": {"S": "jp"},
+ "Upper": {"S": "upper"}, "1abc": {"S": "digit"}, "_u": {"S": "underscore"}, "l_empty": {"L": []},
+ "nan": {"S": "kw nan"}, "false": {"S": "kw false"}}
+ """);
+
+ assertEquals("""
+ $ion_1_0 {Item:{pk:"types",sk:1.,b:{{AAEC/w==}},n_lead:7.,ns:$dynamodb_NS::[2.5,1d3,3.,1.],\
+ 'null':"kw null",'true':"kw true",s_unicode:"日本語 é 😀",\
+ l:["x",1.,true,null,[],{},{a:1.},$dynamodb_SS::["z"]],s_empty:"",m:{'a-b':1.,nested:{deeper:"v"}},\
+ bool_t:true,n_int:103.,bs:$dynamodb_BS::[{{AA==}},{{eA==}}],s:"hello",'a-b':"dash",$x:"dollar",\
+ n_zero:0.,n_38:12345678901234567890123456789012345678.,n_trail:123.45,\
+ s_quote:"he said \\"hi\\"\\n\\ttab \\\\ back 'single'",n_point:1d1,n_2000:2d3,m_empty:{},\
+ ss:$dynamodb_SS::["a","b","c d"],n_big:1d125,bool_f:false,'$10':"dollar digits",'with space':"sp",\
+ n_frac:0.5,nul:null,n_small:1d-130,n_600:6d2,'\\'q':"quote",b_empty:{{}},n_negfrac:-0.001,n_neg:-1.,\
+ n_e:15d2,'+inf':"inf",s_ctrl:"\\x01\\x1f\\x7f\\x80",'a.b':2.,'日本':"jp",Upper:"upper",'1abc':"digit",\
+ _u:"underscore",l_empty:[],'nan':"kw nan",'false':"kw false"}}""", DynamoDbIonText.exportLine(item));
+ }
+
+ @Test
+ void exportLineIsWhatIonJavaWrites() throws IOException {
+ JsonNode item = MAPPER.readTree("""
+ {"pk": {"S": "edge"}, "$": {"S": "lone dollar"}, "$0": {"S": "symbol id"}, "nulls": {"S": "no keyword"},
+ "NULL": {"S": "upper case"}, "+-": {"S": "operators"}, "tab\\tname": {"S": "t"},
+ "quote\\"name": {"S": "q"}, "back\\\\slash": {"S": "b"}, "é": {"S": "latin"}, "😀": {"S": "emoji"},
+ "ctrl": {"S": "\\u0000\\u0007\\b\\t\\n\\u000b\\f\\r\\u001b\\u007f\\u009f\\u00a0\\u00ff\\u0100 😀 ' \\""},
+ "numbers": {"NS": ["0", "-0", "0.000001", "0.0000001", "123456.789", "-1E+125", "1E-130", "-1E-130",
+ "9.9999999999999999999999999999999999999E+125", "100", "0.1", "12.0", "-12.5"]},
+ "nested": {"L": [{"L": [{"M": {"x y": {"SS": ["'", "\\""]}}}]}, {"BS": ["", "AAEC/w=="]}]},
+ "blob": {"B": "////"}}
+ """);
+
+ assertEquals(ionJava(item), DynamoDbIonText.exportLine(item));
+ }
+
+ private static String ionJava(JsonNode item) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (IonWriter writer = IonTextWriterBuilder.standard().withInitialIvmHandling(InitialIvmHandling.ENSURE).build(out)) {
+ writer.stepIn(IonType.STRUCT);
+ writer.setFieldName("Item");
+ writeStruct(writer, item);
+ writer.stepOut();
+ }
+ return out.toString(StandardCharsets.UTF_8);
+ }
+
+ private static void writeStruct(IonWriter writer, JsonNode attributes) throws IOException {
+ writer.stepIn(IonType.STRUCT);
+ for (Map.Entry<String, JsonNode> attribute : attributes.properties()) {
+ writer.setFieldName(attribute.getKey());
+ writeValue(writer, attribute.getValue());
+ }
+ writer.stepOut();
+ }
+
+ private static void writeValue(IonWriter writer, JsonNode value) throws IOException {
+ Map.Entry<String, JsonNode> typed = value.properties().iterator().next();
+ JsonNode content = typed.getValue();
+ switch (typed.getKey()) {
+ case "S" -> writer.writeString(content.asText());
+ case "N" -> writer.writeDecimal(new BigDecimal(content.asText()).stripTrailingZeros());
+ case "B" -> writer.writeBlob(Base64.getDecoder().decode(content.asText()));
+ case "BOOL" -> writer.writeBool(content.asBoolean());
+ case "NULL" -> writer.writeNull();
+ case "SS", "NS", "BS" -> {
+ writer.setTypeAnnotations("$dynamodb_" + typed.getKey());
+ writer.stepIn(IonType.LIST);
+ for (JsonNode member : content) {
+ writeValue(writer, MAPPER.createObjectNode().set(typed.getKey().substring(0, 1), member));
+ }
+ writer.stepOut();
+ }
+ case "L" -> {
+ writer.stepIn(IonType.LIST);
+ for (JsonNode member : content) {
+ writeValue(writer, member);
+ }
+ writer.stepOut();
+ }
+ case "M" -> writeStruct(writer, content);
+ default -> throw new IllegalArgumentException(typed.getKey());
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index d7951ec36b..261fe847e3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -49,6 +49,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.*;
@@ -4532,11 +4533,7 @@ class DynamoDbServiceTest {
@Test
void exportPutsItsFilesUnderThePrefix() {
- S3Service s3 = mock(S3Service.class);
- when(s3.initiateMultipartUpload(eq("bucket"), anyString(), eq("application/octet-stream")))
- .thenAnswer(invocation -> new MultipartUpload("bucket", invocation.getArgument(1), "application/octet-stream"));
- when(s3.completeMultipartUpload(eq("bucket"), anyString(), anyString(), eq(List.of(1)), isNull(), isNull()))
- .thenReturn(new S3Object("bucket", "file", new byte[0], "application/octet-stream", "\"0123-1\""));
+ S3Service s3 = exportBucket();
DynamoDbService svc = inlineExportService(s3);
String longest = "a".repeat(948);
@@ -4554,6 +4551,43 @@ class DynamoDbServiceTest {
.mapToInt(key -> key.getBytes(StandardCharsets.UTF_8).length).max().orElseThrow());
}
+ @Test
+ void ionExportWritesIonLines() throws Exception {
+ S3Service s3 = exportBucket();
+ DynamoDbService svc = inlineExportService(s3);
+ svc.putItem("Users", mapper.readTree("{\"userId\": {\"S\": \"u1\"}, \"total\": {\"N\": \"150\"}}"), "us-east-1");
+
+ String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", "exports-ion",
+ "ExportFormat", "ION"), null, "us-east-1").getExportArn();
+
+ ExportDescription described = svc.describeExport(exportArn);
+ assertEquals("COMPLETED", described.getExportStatus());
+ assertEquals("ION", described.getExportFormat());
+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<byte[]> parts = ArgumentCaptor.forClass(byte[].class);
+ verify(s3, atLeastOnce()).uploadPart(eq("bucket"), keys.capture(), anyString(), eq(1), parts.capture());
+ String dir = "exports-ion/AWSDynamoDB/" + exportArn.substring(exportArn.lastIndexOf('/') + 1) + "/";
+ List<String> uploadedKeys = keys.getAllValues();
+ String dataKey = uploadedKeys.stream().filter(key -> key.endsWith(".ion.gz")).findFirst().orElseThrow();
+ assertTrue(dataKey.matches(dir + "data/[a-z2-7]{26}\\.ion\\.gz"), dataKey);
+ byte[] dataFile = parts.getAllValues().get(uploadedKeys.indexOf(dataKey));
+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(dataFile))) {
+ assertEquals("$ion_1_0 {Item:{userId:\"u1\",total:15d1}}\n", new String(gzip.readAllBytes(), StandardCharsets.UTF_8));
+ }
+ byte[] summary = parts.getAllValues().get(uploadedKeys.indexOf(dir + "manifest-summary.json"));
+ assertEquals("ION", mapper.readTree(summary).get("outputFormat").asText());
+ }
+
+ /** A bucket that takes every upload, each completing with a one-part ETag. */
+ private static S3Service exportBucket() {
+ S3Service s3 = mock(S3Service.class);
+ when(s3.initiateMultipartUpload(eq("bucket"), anyString(), eq("application/octet-stream")))
+ .thenAnswer(invocation -> new MultipartUpload("bucket", invocation.getArgument(1), "application/octet-stream"));
+ when(s3.completeMultipartUpload(eq("bucket"), anyString(), anyString(), eq(List.of(1)), isNull(), isNull()))
+ .thenReturn(new S3Object("bucket", "file", new byte[0], "application/octet-stream", "\"0123-1\""));
+ return s3;
+ }
+
/** Runs each export on the calling thread, with Users created and point-in-time recovery on. */
private DynamoDbService inlineExportService(S3Service s3) {
DynamoDbService svc = new DynamoDbService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),