Light, fluffy, and always free - The AWS Local Emulator alternative
feat(dynamodb): export tables in the ION format (#6429)
TATSUNO “Taz” Yasuhiroed21e67cd46a
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.javaindex 3245c62929..5545cff16c 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/DynamoDbExportTest.java@@ -3,7 +3,6 @@ package com.floci.test;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import org.junit.jupiter.api.*;-import software.amazon.awssdk.core.ResponseBytes;import software.amazon.awssdk.services.dynamodb.DynamoDbClient;import software.amazon.awssdk.services.dynamodb.model.*;import software.amazon.awssdk.services.dynamodb.waiters.DynamoDbWaiter;@@ -178,34 +177,18 @@ class DynamoDbExportTest {void s3Objects_manifestAndDataExist() throws Exception {assertThat(exportArn).isNotNull();- DescribeExportResponse descResp = ddb.describeExport(- DescribeExportRequest.builder().exportArn(exportArn).build());- String manifestSummaryKey = descResp.exportDescription().exportManifest();- assertThat(manifestSummaryKey).isNotBlank();-- // Verify manifest-summary.json exists- ResponseBytes<GetObjectResponse> manifestSummary = s3.getObjectAsBytes(- GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestSummaryKey).build());- assertThat(new ObjectMapper().readTree(manifestSummary.asByteArray()).get("outputFormat").asText())- .isEqualTo("DYNAMODB_JSON");-+ ExportFiles files = readExportFiles(exportArn);+ assertThat(files.manifestSummary().get("outputFormat").asText()).isEqualTo("DYNAMODB_JSON");String exportId = exportArn.substring(exportArn.lastIndexOf('/') + 1);- String manifestFilesKey = "exports/AWSDynamoDB/" + exportId + "/manifest-files.json";+ assertThat(files.manifestSummary().get("manifestFilesS3Key").asText())+ .isEqualTo("exports/AWSDynamoDB/" + exportId + "/manifest-files.json");- ResponseBytes<GetObjectResponse> manifestFiles = s3.getObjectAsBytes(- GetObjectRequest.builder().bucket(BUCKET_NAME).key(manifestFilesKey).build());- JsonNode dataFileEntry = new ObjectMapper().readTree(manifestFiles.asByteArray());- String dataKey = dataFileEntry.get("dataFileS3Key").asText();+ String dataKey = files.dataFileEntry().get("dataFileS3Key").asText();assertThat(dataKey).endsWith(".json.gz");HeadObjectResponse head = s3.headObject(HeadObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());- assertThat(head.eTag()).isEqualTo("\"" + dataFileEntry.get("etag").asText() + "\"").endsWith("-1\"");+ assertThat(head.eTag()).isEqualTo("\"" + files.dataFileEntry().get("etag").asText() + "\"").endsWith("-1\"");- // Download and decompress the data file- ResponseBytes<GetObjectResponse> dataFile = s3.getObjectAsBytes(- GetObjectRequest.builder().bucket(BUCKET_NAME).key(dataKey).build());-- String ndjson = decompressGzip(dataFile.asByteArray());- String[] lines = ndjson.split("\n");+ String[] lines = files.data().split("\n");assertThat(lines).hasSize(3);for (String line : lines) {@@ -277,6 +260,47 @@ class DynamoDbExportTest {.hasMessageContaining("Export Time is invalid");}+ @Test+ @Order(10)+ void exportTableToPointInTime_ionFormat_writesIonLines() throws Exception {+ ExportDescription desc = ddb.exportTableToPointInTime(ExportTableToPointInTimeRequest.builder()+ .tableArn(tableArn)+ .s3Bucket(BUCKET_NAME)+ .s3Prefix("exports-ion")+ .exportFormat(ExportFormat.ION)+ .build()).exportDescription();+ assertThat(desc.exportFormat()).isEqualTo(ExportFormat.ION);++ try (DynamoDbWaiter waiter = DynamoDbWaiter.builder().client(ddb).build()) {+ waiter.waitUntilExportCompleted(r -> r.exportArn(desc.exportArn()));+ }+ ExportFiles files = readExportFiles(desc.exportArn());+ assertThat(files.manifestSummary().get("outputFormat").asText()).isEqualTo("ION");+ assertThat(files.dataFileEntry().get("dataFileS3Key").asText()).endsWith(".ion.gz");+ assertThat(files.data().split("\n")).hasSize(4)+ .allSatisfy(line -> assertThat(line).startsWith("$ion_1_0 {Item:{").endsWith("}}"))+ .anySatisfy(line -> assertThat(line).contains("pk:\"user-3\"", "sk:\"order-003\"", "total:15d1"));+ }++ /** The manifests and the data of an export that wrote one data file. */+ private record ExportFiles(JsonNode manifestSummary, JsonNode dataFileEntry, String data) {}++ private ExportFiles readExportFiles(String arn) throws Exception {+ JsonNode manifestSummary = readJson(ddb.describeExport(DescribeExportRequest.builder().exportArn(arn).build())+ .exportDescription().exportManifest());+ JsonNode dataFileEntry = readJson(manifestSummary.get("manifestFilesS3Key").asText());+ String data = decompressGzip(getObject(dataFileEntry.get("dataFileS3Key").asText()));+ return new ExportFiles(manifestSummary, dataFileEntry, data);+ }++ private JsonNode readJson(String key) throws Exception {+ return new ObjectMapper().readTree(getObject(key));+ }++ private byte[] getObject(String key) {+ return s3.getObjectAsBytes(GetObjectRequest.builder().bucket(BUCKET_NAME).key(key).build()).asByteArray();+ }+private String decompressGzip(byte[] data) throws Exception {try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(data));BufferedReader reader = new BufferedReader(new InputStreamReader(gzip, StandardCharsets.UTF_8))) {diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.mdindex 9ae37f1a5d..0f27b2dd16 100644--- a/docs/services/dynamodb.md+++ b/docs/services/dynamodb.md@@ -409,7 +409,7 @@ Deviations from AWS:## Export to S3-Export table data to an S3 bucket as gzip-compressed NDJSON (DynamoDB JSON format). As on AWS, the table needs point-in-time recovery:+Export table data to an S3 bucket as gzip-compressed lines of DynamoDB JSON (`DYNAMODB_JSON`, the default) or Amazon Ion text (`ION`). As on AWS, the table needs point-in-time recovery:```bash# Create a bucket to receive the export@@ -440,7 +440,7 @@ aws dynamodb list-exports \--endpoint-url $AWS_ENDPOINT_URL```-The export writes `_started`, `manifest-summary.json`, `manifest-files.json` and an `.md5` file for each manifest to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/`, and the items to a `.json.gz` file in its `data/` folder, the layout AWS uses. AWS writes one data file per partition. Floci writes one. As on AWS, every file is a multipart upload of one part, so its ETag ends in `-1`. As on AWS, an `S3Prefix` shorter than 3 characters or longer than 948 UTF-8 bytes is refused, and no slash is added after a prefix that ends with one.+The export writes `_started`, `manifest-summary.json`, `manifest-files.json` and an `.md5` file for each manifest to `s3://<bucket>/<prefix>/AWSDynamoDB/<exportId>/`, and the items to a `.json.gz` or `.ion.gz` file in its `data/` folder, the layout AWS uses. AWS writes one data file per partition. Floci writes one. An Ion line is `$ion_1_0 {Item:{...}}`, with sets annotated as `$dynamodb_SS`, `$dynamodb_NS` and `$dynamodb_BS`, and numbers as Ion decimals. As on AWS, every file is a multipart upload of one part, so its ETag ends in `-1`. As on AWS, an `S3Prefix` shorter than 3 characters or longer than 948 UTF-8 bytes is refused, and no slash is added after a prefix that ends with one.The export holds the items as they were at `ExportTime`, or at the request time when `ExportTime` is not set. As on AWS, `ExportTime` must not be before `EarliestRestorableDateTime` or the table creation, and must not be in the future. A time after `LatestRestorableDateTime` is accepted.diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.javanew file mode 100644index 0000000000..05161b50a1--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonText.java@@ -0,0 +1,140 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;++import java.math.BigDecimal;+import java.util.HexFormat;+import java.util.Map;+import java.util.Set;+import java.util.function.BiConsumer;+import java.util.regex.Pattern;++/**+ * Writes items as the Ion text of an ION export. AWS prints them with the standard text writer of+ * ion-java, so this follows its quoting, escapes and decimal forms.+ */+final class DynamoDbIonText {++ private static final Pattern IDENTIFIER = Pattern.compile("[A-Za-z_$][A-Za-z0-9_$]*");+ /** Ion reads these as symbol ids, so they are quoted like the keywords. */+ private static final Pattern SYMBOL_ID = Pattern.compile("\\$[0-9]+");+ private static final Set<String> KEYWORDS = Set.of("null", "true", "false", "nan");++ private DynamoDbIonText() {}++ /** One line of a full export, without its newline. */+ static String exportLine(JsonNode item) {+ StringBuilder out = new StringBuilder("$ion_1_0 {Item:");+ appendStruct(out, item);+ return out.append('}').toString();+ }++ private static void appendStruct(StringBuilder out, JsonNode attributes) {+ out.append('{');+ String separator = "";+ for (Map.Entry<String, JsonNode> attribute : attributes.properties()) {+ out.append(separator);+ separator = ",";+ appendFieldName(out, attribute.getKey());+ out.append(':');+ appendValue(out, attribute.getValue());+ }+ out.append('}');+ }++ private static void appendValue(StringBuilder out, JsonNode value) {+ Map.Entry<String, JsonNode> typed = value.properties().iterator().next();+ JsonNode content = typed.getValue();+ switch (typed.getKey()) {+ case "S" -> appendString(out, content);+ case "N" -> appendDecimal(out, content);+ case "B" -> appendBlob(out, content);+ case "BOOL" -> out.append(content.asBoolean());+ case "NULL" -> out.append("null");+ case "SS" -> appendList(out.append("$dynamodb_SS::"), content, DynamoDbIonText::appendString);+ case "NS" -> appendList(out.append("$dynamodb_NS::"), content, DynamoDbIonText::appendDecimal);+ case "BS" -> appendList(out.append("$dynamodb_BS::"), content, DynamoDbIonText::appendBlob);+ case "L" -> appendList(out, content, DynamoDbIonText::appendValue);+ case "M" -> appendStruct(out, content);+ default -> throw new IllegalArgumentException("Unknown attribute type: " + typed.getKey());+ }+ }++ private static void appendList(StringBuilder out, JsonNode members, BiConsumer<StringBuilder, JsonNode> appendMember) {+ out.append('[');+ String separator = "";+ for (JsonNode member : members) {+ out.append(separator);+ separator = ",";+ appendMember.accept(out, member);+ }+ out.append(']');+ }++ private static void appendFieldName(StringBuilder out, String name) {+ if (IDENTIFIER.matcher(name).matches() && !KEYWORDS.contains(name) && !SYMBOL_ID.matcher(name).matches()) {+ out.append(name);+ } else {+ out.append('\'');+ appendEscaped(out, name, '\'');+ out.append('\'');+ }+ }++ private static void appendString(StringBuilder out, JsonNode text) {+ out.append('"');+ appendEscaped(out, text.asText(), '"');+ out.append('"');+ }++ private static void appendEscaped(StringBuilder out, String text, char quote) {+ for (int i = 0; i < text.length(); i++) {+ char c = text.charAt(i);+ switch (c) {+ case 0 -> out.append("\\0");+ case 7 -> out.append("\\a");+ case '\b' -> out.append("\\b");+ case '\t' -> out.append("\\t");+ case '\n' -> out.append("\\n");+ case 0x0B -> out.append("\\v");+ case '\f' -> out.append("\\f");+ case '\r' -> out.append("\\r");+ case '\\' -> out.append("\\\\");+ default -> {+ if (c == quote) {+ out.append('\\').append(c);+ } else if (c < 0x20 || (0x7F <= c && c <= 0x9F)) {+ out.append("\\x").append(HexFormat.of().toHexDigits((byte) c));+ } else {+ out.append(c);+ }+ }+ }+ }+ }++ private static void appendDecimal(StringBuilder out, JsonNode number) {+ BigDecimal value = new BigDecimal(number.asText()).stripTrailingZeros();+ if (value.signum() < 0) {+ out.append('-');+ }+ String digits = value.unscaledValue().abs().toString();+ int scale = value.scale();+ if (scale < 0) {+ out.append(digits).append('d').append(-scale);+ } else if (scale == 0) {+ out.append(digits).append('.');+ } else if (scale < digits.length()) {+ int whole = digits.length() - scale;+ out.append(digits, 0, whole).append('.').append(digits, whole, digits.length());+ } else if (scale <= digits.length() + 5) {+ out.append("0.").append("0".repeat(scale - digits.length())).append(digits);+ } else {+ out.append(digits).append("d-").append(scale);+ }+ }++ private static void appendBlob(StringBuilder out, JsonNode base64) {+ out.append("{{").append(base64.asText()).append("}}");+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 2e3bd8aa76..d468b285a6 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -4918,37 +4918,39 @@ public class DynamoDbService {// --- Export Operations ---+ private static final List<String> EXPORT_FORMATS = List.of("ION", "DYNAMODB_JSON");+/** As on AWS, an ExportTime after LatestRestorableDateTime is accepted up to now. */public ExportDescription exportTable(Map<String, Object> request, Instant exportTime, String region) {String tableArn = (String) request.get("TableArn");String s3Bucket = (String) request.get("S3Bucket");String s3Prefix = request.containsKey("S3Prefix") ? (String) request.get("S3Prefix") : null;- String exportFormat = request.containsKey("ExportFormat") ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";+ String exportFormat = request.get("ExportFormat") != null ? (String) request.get("ExportFormat") : "DYNAMODB_JSON";String exportType = request.containsKey("ExportType") ? (String) request.get("ExportType") : "FULL_EXPORT";String clientToken = request.containsKey("ClientToken") ? (String) request.get("ClientToken") : null;String s3SseAlgorithm = request.containsKey("S3SseAlgorithm") ? (String) request.get("S3SseAlgorithm") : null;String s3BucketOwner = request.containsKey("S3BucketOwner") ? (String) request.get("S3BucketOwner") : null;- if (s3Prefix != null) {- int length = s3Prefix.codePointCount(0, s3Prefix.length());- if (1024 < length) {- throw SmithyValidationErrors.validationException("Value '" + s3Prefix- + "' at 's3Prefix' failed to satisfy constraint: Member must have length less than or equal to 1024");- }- if (length < 3) {- throw new AwsException("ValidationException", "Invalid Request: " + SmithyValidationErrors.message(- List.of("Value '" + s3Prefix + "' at 's3Prefix' failed to satisfy constraint: "- + "Member must have length greater than or equal to 3")), 400);- }+ List<String> violations = new ArrayList<>();+ if (s3Prefix != null && 1024 < s3Prefix.codePointCount(0, s3Prefix.length())) {+ violations.add("Value '" + s3Prefix+ + "' at 's3Prefix' failed to satisfy constraint: Member must have length less than or equal to 1024");+ }+ if (!EXPORT_FORMATS.contains(exportFormat)) {+ violations.add(SmithyValidationErrors.notInEnum(exportFormat, "exportFormat", EXPORT_FORMATS));+ }+ if (!violations.isEmpty()) {+ throw SmithyValidationErrors.validationException(violations);+ }+ if (s3Prefix != null && s3Prefix.codePointCount(0, s3Prefix.length()) < 3) {+ throw new AwsException("ValidationException", "Invalid Request: " + SmithyValidationErrors.message(+ List.of("Value '" + s3Prefix + "' at 's3Prefix' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 3")), 400);}if ("INCREMENTAL_EXPORT".equals(exportType)) {throw new AwsException("ValidationException","ExportType INCREMENTAL_EXPORT is not supported", 400);}- if ("ION".equals(exportFormat)) {- throw new AwsException("ValidationException",- "ExportFormat ION is not supported", 400);- }DynamoDbTableNames.ResolvedTableRef ref = DynamoDbTableNames.resolveWithRegion(tableArn, region);String tableName = ref.name();@@ -5037,8 +5039,9 @@ public class DynamoDbService {throw e;}- byte[] gzipData = buildGzipNdjson(snapshot);- String dataKey = exportDir + "data/" + exportFileName() + ".json.gz";+ boolean ion = "ION".equals(desc.getExportFormat());+ byte[] gzipData = buildGzipExportData(snapshot, ion);+ String dataKey = exportDir + "data/" + exportFileName() + (ion ? ".ion.gz" : ".json.gz");ObjectNode dataFile = objectMapper.createObjectNode();dataFile.put("itemCount", snapshot.size());dataFile.put("md5Checksum", Base64.getEncoder().encodeToString(Digests.md5(gzipData)));@@ -5072,14 +5075,18 @@ public class DynamoDbService {}}- private byte[] buildGzipNdjson(List<JsonNode> items) throws IOException {+ private byte[] buildGzipExportData(List<JsonNode> items, boolean ion) throws IOException {ByteArrayOutputStream baos = new ByteArrayOutputStream();try (GZIPOutputStream gzip = new GZIPOutputStream(baos)) {for (JsonNode item : items) {- ObjectNode line = objectMapper.createObjectNode();- line.set("Item", item);- byte[] lineBytes = objectMapper.writeValueAsBytes(line);- gzip.write(lineBytes);+ if (ion) {+ gzip.write(DynamoDbIonText.exportLine(item).getBytes(StandardCharsets.UTF_8));+ } else {+ ObjectNode line = objectMapper.createObjectNode();+ line.set("Item", item);+ byte[] lineBytes = objectMapper.writeValueAsBytes(line);+ gzip.write(lineBytes);+ }gzip.write('\n');}}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.javaindex 1f7b0d95ac..6ec9aef238 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportIntegrationTest.java@@ -176,21 +176,23 @@ class DynamoDbExportIntegrationTest {}@Test- void exportTableToPointInTime_ionFormat_returnsValidationException() {- given()- .header("X-Amz-Target", "DynamoDB_20120810.ExportTableToPointInTime")- .contentType(DYNAMODB_CONTENT_TYPE)- .body("""- {- "TableArn": "%s",- "S3Bucket": "%s",- "ExportFormat": "ION"- }- """.formatted(TABLE_ARN, BUCKET_NAME))- .when().post("/")- .then()- .statusCode(400)- .body("__type", containsString("ValidationException"));+ void exportFormatIsCheckedAsAwsChecksIt() {+ String enumSet = "' at 'exportFormat' failed to satisfy constraint: "+ + "Member must satisfy enum value set: [ION, DYNAMODB_JSON]";+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "ExportFormat": "ion", "S3Prefix": "x"}+ """.formatted(TABLE_ARN_PREFIX + "ExportFormatMissing", BUCKET_NAME)), "ValidationException",+ "1 validation error detected: Value 'ion" + enumSet);+ String over1024 = "a".repeat(1025);+ expectError(ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "ExportFormat": "", "S3Prefix": "%s"}+ """.formatted(TABLE_ARN, BUCKET_NAME, over1024)), "ValidationException",+ "2 validation errors detected: Value '" + over1024 + "' at 's3Prefix' failed to satisfy constraint: "+ + "Member must have length less than or equal to 1024; Value '" + enumSet);+ ddb("ExportTableToPointInTime", """+ {"TableArn": "%s", "S3Bucket": "%s", "S3Prefix": "exports-ion", "ExportFormat": "ION"}+ """.formatted(TABLE_ARN, BUCKET_NAME)).statusCode(200)+ .body("ExportDescription.ExportFormat", equalTo("ION"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.javanew file mode 100644index 0000000000..90443c11b2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIonTextTest.java@@ -0,0 +1,124 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.amazon.ion.IonType;+import com.amazon.ion.IonWriter;+import com.amazon.ion.system.IonTextWriterBuilder;+import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.math.BigDecimal;+import java.nio.charset.StandardCharsets;+import java.util.Base64;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;++class DynamoDbIonTextTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ @Test+ void exportLineIsTheLineAwsWrites() throws IOException {+ JsonNode item = MAPPER.readTree("""+ {"pk": {"S": "types"}, "sk": {"N": "1"}, "b": {"B": "AAEC/w=="}, "n_lead": {"N": "007"},+ "ns": {"NS": ["2.50", "1E+3", "3", "1"]}, "null": {"S": "kw null"}, "true": {"S": "kw true"},+ "s_unicode": {"S": "日本語 é 😀"},+ "l": {"L": [{"S": "x"}, {"N": "1"}, {"BOOL": true}, {"NULL": true}, {"L": []}, {"M": {}},+ {"M": {"a": {"N": "1"}}}, {"SS": ["z"]}]},+ "s_empty": {"S": ""}, "m": {"M": {"a-b": {"N": "1"}, "nested": {"M": {"deeper": {"S": "v"}}}}},+ "bool_t": {"BOOL": true}, "n_int": {"N": "103"}, "bs": {"BS": ["AA==", "eA=="]}, "s": {"S": "hello"},+ "a-b": {"S": "dash"}, "$x": {"S": "dollar"}, "n_zero": {"N": "0"},+ "n_38": {"N": "12345678901234567890123456789012345678"}, "n_trail": {"N": "123.4500"},+ "s_quote": {"S": "he said \\"hi\\"\\n\\ttab \\\\ back 'single'"}, "n_point": {"N": "10.0"},+ "n_2000": {"N": "2000"}, "m_empty": {"M": {}}, "ss": {"SS": ["a", "b", "c d"]},+ "n_big": {"N": "1E+125"}, "bool_f": {"BOOL": false}, "$10": {"S": "dollar digits"},+ "with space": {"S": "sp"}, "n_frac": {"N": "0.5"}, "nul": {"NULL": true}, "n_small": {"N": "1E-130"},+ "n_600": {"N": "600"}, "'q": {"S": "quote"}, "b_empty": {"B": ""}, "n_negfrac": {"N": "-0.001"},+ "n_neg": {"N": "-1"}, "n_e": {"N": "1.5e3"}, "+inf": {"S": "inf"},+ "s_ctrl": {"S": "\\u0001\\u001f\\u007f\\u0080"}, "a.b": {"N": "2"}, "日本": {"S": "jp"},+ "Upper": {"S": "upper"}, "1abc": {"S": "digit"}, "_u": {"S": "underscore"}, "l_empty": {"L": []},+ "nan": {"S": "kw nan"}, "false": {"S": "kw false"}}+ """);++ assertEquals("""+ $ion_1_0 {Item:{pk:"types",sk:1.,b:{{AAEC/w==}},n_lead:7.,ns:$dynamodb_NS::[2.5,1d3,3.,1.],\+ 'null':"kw null",'true':"kw true",s_unicode:"日本語 é 😀",\+ l:["x",1.,true,null,[],{},{a:1.},$dynamodb_SS::["z"]],s_empty:"",m:{'a-b':1.,nested:{deeper:"v"}},\+ bool_t:true,n_int:103.,bs:$dynamodb_BS::[{{AA==}},{{eA==}}],s:"hello",'a-b':"dash",$x:"dollar",\+ n_zero:0.,n_38:12345678901234567890123456789012345678.,n_trail:123.45,\+ s_quote:"he said \\"hi\\"\\n\\ttab \\\\ back 'single'",n_point:1d1,n_2000:2d3,m_empty:{},\+ ss:$dynamodb_SS::["a","b","c d"],n_big:1d125,bool_f:false,'$10':"dollar digits",'with space':"sp",\+ n_frac:0.5,nul:null,n_small:1d-130,n_600:6d2,'\\'q':"quote",b_empty:{{}},n_negfrac:-0.001,n_neg:-1.,\+ n_e:15d2,'+inf':"inf",s_ctrl:"\\x01\\x1f\\x7f\\x80",'a.b':2.,'日本':"jp",Upper:"upper",'1abc':"digit",\+ _u:"underscore",l_empty:[],'nan':"kw nan",'false':"kw false"}}""", DynamoDbIonText.exportLine(item));+ }++ @Test+ void exportLineIsWhatIonJavaWrites() throws IOException {+ JsonNode item = MAPPER.readTree("""+ {"pk": {"S": "edge"}, "$": {"S": "lone dollar"}, "$0": {"S": "symbol id"}, "nulls": {"S": "no keyword"},+ "NULL": {"S": "upper case"}, "+-": {"S": "operators"}, "tab\\tname": {"S": "t"},+ "quote\\"name": {"S": "q"}, "back\\\\slash": {"S": "b"}, "é": {"S": "latin"}, "😀": {"S": "emoji"},+ "ctrl": {"S": "\\u0000\\u0007\\b\\t\\n\\u000b\\f\\r\\u001b\\u007f\\u009f\\u00a0\\u00ff\\u0100 😀 ' \\""},+ "numbers": {"NS": ["0", "-0", "0.000001", "0.0000001", "123456.789", "-1E+125", "1E-130", "-1E-130",+ "9.9999999999999999999999999999999999999E+125", "100", "0.1", "12.0", "-12.5"]},+ "nested": {"L": [{"L": [{"M": {"x y": {"SS": ["'", "\\""]}}}]}, {"BS": ["", "AAEC/w=="]}]},+ "blob": {"B": "////"}}+ """);++ assertEquals(ionJava(item), DynamoDbIonText.exportLine(item));+ }++ private static String ionJava(JsonNode item) throws IOException {+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ try (IonWriter writer = IonTextWriterBuilder.standard().withInitialIvmHandling(InitialIvmHandling.ENSURE).build(out)) {+ writer.stepIn(IonType.STRUCT);+ writer.setFieldName("Item");+ writeStruct(writer, item);+ writer.stepOut();+ }+ return out.toString(StandardCharsets.UTF_8);+ }++ private static void writeStruct(IonWriter writer, JsonNode attributes) throws IOException {+ writer.stepIn(IonType.STRUCT);+ for (Map.Entry<String, JsonNode> attribute : attributes.properties()) {+ writer.setFieldName(attribute.getKey());+ writeValue(writer, attribute.getValue());+ }+ writer.stepOut();+ }++ private static void writeValue(IonWriter writer, JsonNode value) throws IOException {+ Map.Entry<String, JsonNode> typed = value.properties().iterator().next();+ JsonNode content = typed.getValue();+ switch (typed.getKey()) {+ case "S" -> writer.writeString(content.asText());+ case "N" -> writer.writeDecimal(new BigDecimal(content.asText()).stripTrailingZeros());+ case "B" -> writer.writeBlob(Base64.getDecoder().decode(content.asText()));+ case "BOOL" -> writer.writeBool(content.asBoolean());+ case "NULL" -> writer.writeNull();+ case "SS", "NS", "BS" -> {+ writer.setTypeAnnotations("$dynamodb_" + typed.getKey());+ writer.stepIn(IonType.LIST);+ for (JsonNode member : content) {+ writeValue(writer, MAPPER.createObjectNode().set(typed.getKey().substring(0, 1), member));+ }+ writer.stepOut();+ }+ case "L" -> {+ writer.stepIn(IonType.LIST);+ for (JsonNode member : content) {+ writeValue(writer, member);+ }+ writer.stepOut();+ }+ case "M" -> writeStruct(writer, content);+ default -> throw new IllegalArgumentException(typed.getKey());+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex d7951ec36b..261fe847e3 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -49,6 +49,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;+import java.util.zip.GZIPInputStream;import java.util.zip.GZIPOutputStream;import static org.junit.jupiter.api.Assertions.*;@@ -4532,11 +4533,7 @@ class DynamoDbServiceTest {@Testvoid exportPutsItsFilesUnderThePrefix() {- S3Service s3 = mock(S3Service.class);- when(s3.initiateMultipartUpload(eq("bucket"), anyString(), eq("application/octet-stream")))- .thenAnswer(invocation -> new MultipartUpload("bucket", invocation.getArgument(1), "application/octet-stream"));- when(s3.completeMultipartUpload(eq("bucket"), anyString(), anyString(), eq(List.of(1)), isNull(), isNull()))- .thenReturn(new S3Object("bucket", "file", new byte[0], "application/octet-stream", "\"0123-1\""));+ S3Service s3 = exportBucket();DynamoDbService svc = inlineExportService(s3);String longest = "a".repeat(948);@@ -4554,6 +4551,43 @@ class DynamoDbServiceTest {.mapToInt(key -> key.getBytes(StandardCharsets.UTF_8).length).max().orElseThrow());}+ @Test+ void ionExportWritesIonLines() throws Exception {+ S3Service s3 = exportBucket();+ DynamoDbService svc = inlineExportService(s3);+ svc.putItem("Users", mapper.readTree("{\"userId\": {\"S\": \"u1\"}, \"total\": {\"N\": \"150\"}}"), "us-east-1");++ String exportArn = svc.exportTable(Map.of("TableArn", USERS_ARN, "S3Bucket", "bucket", "S3Prefix", "exports-ion",+ "ExportFormat", "ION"), null, "us-east-1").getExportArn();++ ExportDescription described = svc.describeExport(exportArn);+ assertEquals("COMPLETED", described.getExportStatus());+ assertEquals("ION", described.getExportFormat());+ ArgumentCaptor<String> keys = ArgumentCaptor.forClass(String.class);+ ArgumentCaptor<byte[]> parts = ArgumentCaptor.forClass(byte[].class);+ verify(s3, atLeastOnce()).uploadPart(eq("bucket"), keys.capture(), anyString(), eq(1), parts.capture());+ String dir = "exports-ion/AWSDynamoDB/" + exportArn.substring(exportArn.lastIndexOf('/') + 1) + "/";+ List<String> uploadedKeys = keys.getAllValues();+ String dataKey = uploadedKeys.stream().filter(key -> key.endsWith(".ion.gz")).findFirst().orElseThrow();+ assertTrue(dataKey.matches(dir + "data/[a-z2-7]{26}\\.ion\\.gz"), dataKey);+ byte[] dataFile = parts.getAllValues().get(uploadedKeys.indexOf(dataKey));+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(dataFile))) {+ assertEquals("$ion_1_0 {Item:{userId:\"u1\",total:15d1}}\n", new String(gzip.readAllBytes(), StandardCharsets.UTF_8));+ }+ byte[] summary = parts.getAllValues().get(uploadedKeys.indexOf(dir + "manifest-summary.json"));+ assertEquals("ION", mapper.readTree(summary).get("outputFormat").asText());+ }++ /** A bucket that takes every upload, each completing with a one-part ETag. */+ private static S3Service exportBucket() {+ S3Service s3 = mock(S3Service.class);+ when(s3.initiateMultipartUpload(eq("bucket"), anyString(), eq("application/octet-stream")))+ .thenAnswer(invocation -> new MultipartUpload("bucket", invocation.getArgument(1), "application/octet-stream"));+ when(s3.completeMultipartUpload(eq("bucket"), anyString(), anyString(), eq(List.of(1)), isNull(), isNull()))+ .thenReturn(new S3Object("bucket", "file", new byte[0], "application/octet-stream", "\"0123-1\""));+ return s3;+ }+/** Runs each export on the calling thread, with Users created and point-in-time recovery on. */private DynamoDbService inlineExportService(S3Service s3) {DynamoDbService svc = new DynamoDbService(new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(),