Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): write a large whole number in full in $string (#2683)
Facundo Paredes2a7ee7a60580
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex bb096666df..30d0edda69 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -90,6 +90,10 @@ Three behaviours are worth knowing before reading an unexpected result, and allobject, so the field goes missing instead of the state failing.- `$range` collapses a single-element range to the bare number, not a one-element array.+JSONata's own `$string` follows AWS's number notation: a whole number is written out in full below+`1e21` and in exponent notation from there, on both signs, so `$string(1e20)` is+`100000000000000000000` and `$string(1e21)` is `1e+21`.+One deviation. AWS bounds the memory an expression may use and fails a `$range` of roughly half amillion elements with `Expression evaluation memory limit exceeded`; Floci has no such bound andbuilds the array.diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.javaindex 5ead99c0b9..a2bcb834ee 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.stepfunctions;+import com.dashjoin.jsonata.Functions;import com.dashjoin.jsonata.JException;import com.dashjoin.jsonata.Jsonata;import com.fasterxml.jackson.core.JsonParser;@@ -14,6 +15,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import java.math.BigDecimal;import java.math.BigInteger;import java.nio.charset.StandardCharsets;import java.security.MessageDigest;@@ -53,6 +55,13 @@ public class JsonataEvaluator {*/private static final double LARGEST_EXACT_LONG_AS_DOUBLE = 9.223372036854776E18;+ /**+ * The magnitude at which AWS stops writing a whole number in full and switches to exponent+ * notation, on both signs: {@code $string(1e20)} is {@code "100000000000000000000"} and+ * {@code $string(1e21)} is {@code "1e+21"}.+ */+ private static final double SMALLEST_EXPONENT_NOTATION = 1e21;+private final ObjectMapper objectMapper;private final ObjectReader strictJsonReader;private final Map<String, Jsonata.JFunction> stepFunctionsExtensions;@@ -197,17 +206,22 @@ public class JsonataEvaluator {}/**- * The six JSONata functions the Step Functions dialect adds on top of the JSONata language,- * per https://docs.aws.amazon.com/step-functions/latest/dg/transforming-data.html. Five of- * them the dashjoin library does not have at all; $random it has with the wrong arity, so the- * binding shadows it to accept AWS's optional seed.+ * The JSONata functions Floci binds on the evaluation frame. Six of them are what the Step+ * Functions dialect adds on top of the JSONata language, per+ * https://docs.aws.amazon.com/step-functions/latest/dg/transforming-data.html: five the+ * dashjoin library does not have at all, and $random it has with the wrong arity, so the+ * binding shadows it to accept AWS's optional seed. The seventh, $string, is a JSONata+ * function the library has but writes with a different number notation than AWS.*- * <p>Each signature declares one optional slot more than the arity AWS documents. The library+ * <p>Each of the six declares one optional slot more than the arity AWS documents. The library* pads a short call with Java nulls up to the declared arity and refuses a call longer than* it, so the extra slot is what lets an over-long call reach this code and fail with AWS's own- * message instead of the library's. A signature is also what makes the function usable as a- * value: dashjoin dereferences it unconditionally in $map, $filter, $sift, $each, $single and- * $reduce, and a null one throws a raw NullPointerException there.+ * message instead of the library's. $string keeps the library's own signature instead, so+ * every arity and type error it already raised stays word for word what it was.+ *+ * <p>A signature is also what makes a function usable as a value: dashjoin dereferences it+ * unconditionally in $map, $filter, $sift, $each, $single and $reduce, and a null one throws a+ * raw NullPointerException there.*/private Map<String, Jsonata.JFunction> buildStepFunctionsExtensions() {return Map.of(@@ -216,7 +230,8 @@ public class JsonataEvaluator {"range", new Jsonata.JFunction((input, arguments) -> range(arguments), "<x?x?x?x?:x>"),"hash", new Jsonata.JFunction((input, arguments) -> hash(arguments), "<x?x?x?:x>"),"random", new Jsonata.JFunction((input, arguments) -> random(arguments), "<x?x?:x>"),- "uuid", new Jsonata.JFunction((input, arguments) -> uuid(arguments), "<x?:x>"));+ "uuid", new Jsonata.JFunction((input, arguments) -> uuid(arguments), "<x?:x>"),+ "string", new Jsonata.JFunction((input, arguments) -> string(arguments), "<x-b?:s>"));}/**@@ -332,6 +347,45 @@ public class JsonataEvaluator {return value;}+ /**+ * $string(value, prettify): the JSONata function, with AWS's number notation. dashjoin writes+ * a whole number in full only while it fits in a long and prints exponent notation from there,+ * so $string(1e20) is "1e+20" where AWS writes the twenty-one digits. AWS's boundary is 1e21.+ *+ * <p>Nothing else about the function changes: the call reaches the library with the numbers+ * AWS writes in full already replaced by their digits, so the delimiters, the escaping, the+ * prettify layout and every error message stay the library's.+ */+ private static Object string(List<Object> arguments) {+ return Functions.string(withWholeNumbersWrittenInFull(argument(arguments, 0)),+ (Boolean) argument(arguments, 1));+ }++ /**+ * Replaces every double AWS writes in full with the exact integer of the shortest decimal that+ * reads back as that double, which is the digit string AWS prints: a double holding 2^63 is+ * "9223372036854776000" there and not its exact value 9223372036854775808. A BigInteger is+ * printed verbatim, so the notation stops being the library's decision.+ */+ private static Object withWholeNumbersWrittenInFull(Object value) {+ if (value instanceof Double number) {+ boolean writtenInFull = Double.isFinite(number) && number % 1 == 0+ && Math.abs(number) < SMALLEST_EXPONENT_NOTATION;+ return writtenInFull ? BigDecimal.valueOf(number).toBigInteger() : number;+ }+ if (value instanceof Map<?, ?> object) {+ Map<String, Object> written = new LinkedHashMap<>();+ object.forEach((key, field) -> written.put(String.valueOf(key), withWholeNumbersWrittenInFull(field)));+ return written;+ }+ if (value instanceof List<?> array) {+ List<Object> written = new ArrayList<>();+ array.forEach(element -> written.add(withWholeNumbersWrittenInFull(element)));+ return written;+ }+ return value;+ }+/*** $partition(array, chunkSize): splits array into chunks of chunkSize elements, the last chunk* holding the remainder. A missing chunk size returns the whole array as one chunk; a missingdiff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.javaindex 5ec2a39fbd..d1dce67b86 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java@@ -549,4 +549,73 @@ class JsonataEvaluatorTest {JsonNode result = evaluator.evaluate("{% $map(['{\"a\":1}', '{\"a\":2}'], $parse).a %}", statesVar);assertEquals("[1,2]", result.toString());}++ @Test+ void string_writesAWholeNumberInFullBelowTheExponentBoundary() {+ // dashjoin stops writing digits at Long.MAX_VALUE, 9.223372036854776E18, and prints+ // exponent notation from there. AWS keeps writing them until 1e21.+ JsonNode statesVar = objectMapper.createObjectNode();+ assertEquals("10000000000000000000", evaluator.evaluate("{% $string(1e19) %}", statesVar).asText());+ assertEquals("100000000000000000000", evaluator.evaluate("{% $string(1e20) %}", statesVar).asText());+ assertEquals("150000000000000000000", evaluator.evaluate("{% $string(1.5e20) %}", statesVar).asText());+ // The largest double under the boundary.+ assertEquals("999999999999999900000",+ evaluator.evaluate("{% $string(999999999999999868928) %}", statesVar).asText());+ }++ @Test+ void string_switchesToExponentNotationAt1e21() {+ JsonNode statesVar = objectMapper.createObjectNode();+ assertEquals("1e+21", evaluator.evaluate("{% $string(1e21) %}", statesVar).asText());+ // 1e21 - 1 is not a double, so it is the same number as 1e21 and prints the same way.+ assertEquals("1e+21", evaluator.evaluate("{% $string(1e21 - 1) %}", statesVar).asText());+ assertEquals("1e+22", evaluator.evaluate("{% $string(1e22) %}", statesVar).asText());+ assertEquals("1e+100", evaluator.evaluate("{% $string(1e100) %}", statesVar).asText());+ }++ @Test+ void string_flipsAtTheSameBoundaryOnANegativeNumber() {+ JsonNode statesVar = objectMapper.createObjectNode();+ assertEquals("-100000000000000000000", evaluator.evaluate("{% $string(-1e20) %}", statesVar).asText());+ assertEquals("-150000000000000000000", evaluator.evaluate("{% $string(-1.5e20) %}", statesVar).asText());+ assertEquals("-1e+21", evaluator.evaluate("{% $string(-1e21) %}", statesVar).asText());+ }++ @Test+ void string_writesTheShortestDigitsThatReadBackAsTheSameDouble() {+ // The two sides of $parse's number model print differently across the long boundary: the+ // long prints its exact value, the double prints the shortest decimal that reads back as+ // itself, so 2^63 is 9223372036854776000 and not its exact 9223372036854775808.+ JsonNode statesVar = objectMapper.createObjectNode();+ assertEquals("9223372036854775807",+ evaluator.evaluate("{% $string($parse('{\"n\":9223372036854775807}').n) %}", statesVar).asText());+ assertEquals("9223372036854776000",+ evaluator.evaluate("{% $string($parse('{\"n\":9223372036854775808}').n) %}", statesVar).asText());+ }++ @Test+ void string_writesANumberInsideAnObjectOrAnArrayTheSameWay() {+ JsonNode statesVar = objectMapper.createObjectNode();+ assertEquals("{\"n\":100000000000000000000}",+ evaluator.evaluate("{% $string({'n': 1e20}) %}", statesVar).asText());+ assertEquals("[100000000000000000000,2]",+ evaluator.evaluate("{% $string([1e20, 2]) %}", statesVar).asText());+ }++ @Test+ void string_leavesEveryOtherValueToTheBuiltIn() {+ JsonNode statesVar = objectMapper.createObjectNode();+ // AWS switches to exponent notation on the small side too, under 1e-6, and keeps fifteen+ // significant digits on a fractional number. Both already match.+ assertEquals("0.000001", evaluator.evaluate("{% $string(1e-6) %}", statesVar).asText());+ assertEquals("1e-7", evaluator.evaluate("{% $string(1e-7) %}", statesVar).asText());+ assertEquals("0.333333333333333", evaluator.evaluate("{% $string(1/3) %}", statesVar).asText());+ assertEquals("true", evaluator.evaluate("{% $string(true) %}", statesVar).asText());+ assertEquals("abc", evaluator.evaluate("{% $string('abc') %}", statesVar).asText());+ assertEquals("null", evaluator.evaluate("{% $string(null) %}", statesVar).asText());+ assertEquals("{\n \"a\": 1\n}", evaluator.evaluate("{% $string({'a': 1}, true) %}", statesVar).asText());+ assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isNull());+ assertEquals("[\"100000000000000000000\",\"1e+21\"]",+ evaluator.evaluate("{% $map([1e20, 1e21], $string) %}", statesVar).toString());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex e53064fd6a..949fc2b1b4 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -52,6 +52,34 @@ class StepFunctionsJsonataIntegrationTest {assertTrue(output.contains("42"));}+ @Test+ void passStateWithJsonataStringOfALargeWholeNumber() throws Exception {+ // $string of an id feeds a MessageGroupId, an S3 key or a DynamoDB key, so exponent+ // notation writes a different key without failing the execution.+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Stringify",+ "States": {+ "Stringify": {+ "Type": "Pass",+ "Output": {+ "underTheBoundary": "{% $string(1e20) %}",+ "atTheBoundary": "{% $string(1e21) %}"+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("jsonata-string-large-whole-number", definition);+ JsonNode output = objectMapper.readTree(waitForExecution(startExecution(smArn, "{}")));++ assertEquals("100000000000000000000", output.get("underTheBoundary").asText());+ assertEquals("1e+21", output.get("atTheBoundary").asText());+ }+@Testvoid choiceStateWithJsonataCondition() throws Exception {// Choice state using JSONata Condition instead of Variable/StringEquals