Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): write a large whole number in full in $string (#2683)

Facundo Paredes2a7ee7a60580
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index bb096666df..30d0edda69 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -90,6 +90,10 @@ Three behaviours are worth knowing before reading an unexpected result, and all
object, so the field goes missing instead of the state failing.
- `$range` collapses a single-element range to the bare number, not a one-element array.
+JSONata's own `$string` follows AWS's number notation: a whole number is written out in full below
+`1e21` and in exponent notation from there, on both signs, so `$string(1e20)` is
+`100000000000000000000` and `$string(1e21)` is `1e+21`.
+
One deviation. AWS bounds the memory an expression may use and fails a `$range` of roughly half a
million elements with `Expression evaluation memory limit exceeded`; Floci has no such bound and
builds the array.
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
index 5ead99c0b9..a2bcb834ee 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.stepfunctions;
+import com.dashjoin.jsonata.Functions;
import com.dashjoin.jsonata.JException;
import com.dashjoin.jsonata.Jsonata;
import com.fasterxml.jackson.core.JsonParser;
@@ -14,6 +15,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
+import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -53,6 +55,13 @@ public class JsonataEvaluator {
*/
private static final double LARGEST_EXACT_LONG_AS_DOUBLE = 9.223372036854776E18;
+ /**
+ * The magnitude at which AWS stops writing a whole number in full and switches to exponent
+ * notation, on both signs: {@code $string(1e20)} is {@code "100000000000000000000"} and
+ * {@code $string(1e21)} is {@code "1e+21"}.
+ */
+ private static final double SMALLEST_EXPONENT_NOTATION = 1e21;
+
private final ObjectMapper objectMapper;
private final ObjectReader strictJsonReader;
private final Map<String, Jsonata.JFunction> stepFunctionsExtensions;
@@ -197,17 +206,22 @@ public class JsonataEvaluator {
}
/**
- * The six JSONata functions the Step Functions dialect adds on top of the JSONata language,
- * per https://docs.aws.amazon.com/step-functions/latest/dg/transforming-data.html. Five of
- * them the dashjoin library does not have at all; $random it has with the wrong arity, so the
- * binding shadows it to accept AWS's optional seed.
+ * The JSONata functions Floci binds on the evaluation frame. Six of them are what the Step
+ * Functions dialect adds on top of the JSONata language, per
+ * https://docs.aws.amazon.com/step-functions/latest/dg/transforming-data.html: five the
+ * dashjoin library does not have at all, and $random it has with the wrong arity, so the
+ * binding shadows it to accept AWS's optional seed. The seventh, $string, is a JSONata
+ * function the library has but writes with a different number notation than AWS.
*
- * <p>Each signature declares one optional slot more than the arity AWS documents. The library
+ * <p>Each of the six declares one optional slot more than the arity AWS documents. The library
* pads a short call with Java nulls up to the declared arity and refuses a call longer than
* it, so the extra slot is what lets an over-long call reach this code and fail with AWS's own
- * message instead of the library's. A signature is also what makes the function usable as a
- * value: dashjoin dereferences it unconditionally in $map, $filter, $sift, $each, $single and
- * $reduce, and a null one throws a raw NullPointerException there.
+ * message instead of the library's. $string keeps the library's own signature instead, so
+ * every arity and type error it already raised stays word for word what it was.
+ *
+ * <p>A signature is also what makes a function usable as a value: dashjoin dereferences it
+ * unconditionally in $map, $filter, $sift, $each, $single and $reduce, and a null one throws a
+ * raw NullPointerException there.
*/
private Map<String, Jsonata.JFunction> buildStepFunctionsExtensions() {
return Map.of(
@@ -216,7 +230,8 @@ public class JsonataEvaluator {
"range", new Jsonata.JFunction((input, arguments) -> range(arguments), "<x?x?x?x?:x>"),
"hash", new Jsonata.JFunction((input, arguments) -> hash(arguments), "<x?x?x?:x>"),
"random", new Jsonata.JFunction((input, arguments) -> random(arguments), "<x?x?:x>"),
- "uuid", new Jsonata.JFunction((input, arguments) -> uuid(arguments), "<x?:x>"));
+ "uuid", new Jsonata.JFunction((input, arguments) -> uuid(arguments), "<x?:x>"),
+ "string", new Jsonata.JFunction((input, arguments) -> string(arguments), "<x-b?:s>"));
}
/**
@@ -332,6 +347,45 @@ public class JsonataEvaluator {
return value;
}
+ /**
+ * $string(value, prettify): the JSONata function, with AWS's number notation. dashjoin writes
+ * a whole number in full only while it fits in a long and prints exponent notation from there,
+ * so $string(1e20) is "1e+20" where AWS writes the twenty-one digits. AWS's boundary is 1e21.
+ *
+ * <p>Nothing else about the function changes: the call reaches the library with the numbers
+ * AWS writes in full already replaced by their digits, so the delimiters, the escaping, the
+ * prettify layout and every error message stay the library's.
+ */
+ private static Object string(List<Object> arguments) {
+ return Functions.string(withWholeNumbersWrittenInFull(argument(arguments, 0)),
+ (Boolean) argument(arguments, 1));
+ }
+
+ /**
+ * Replaces every double AWS writes in full with the exact integer of the shortest decimal that
+ * reads back as that double, which is the digit string AWS prints: a double holding 2^63 is
+ * "9223372036854776000" there and not its exact value 9223372036854775808. A BigInteger is
+ * printed verbatim, so the notation stops being the library's decision.
+ */
+ private static Object withWholeNumbersWrittenInFull(Object value) {
+ if (value instanceof Double number) {
+ boolean writtenInFull = Double.isFinite(number) && number % 1 == 0
+ && Math.abs(number) < SMALLEST_EXPONENT_NOTATION;
+ return writtenInFull ? BigDecimal.valueOf(number).toBigInteger() : number;
+ }
+ if (value instanceof Map<?, ?> object) {
+ Map<String, Object> written = new LinkedHashMap<>();
+ object.forEach((key, field) -> written.put(String.valueOf(key), withWholeNumbersWrittenInFull(field)));
+ return written;
+ }
+ if (value instanceof List<?> array) {
+ List<Object> written = new ArrayList<>();
+ array.forEach(element -> written.add(withWholeNumbersWrittenInFull(element)));
+ return written;
+ }
+ return value;
+ }
+
/**
* $partition(array, chunkSize): splits array into chunks of chunkSize elements, the last chunk
* holding the remainder. A missing chunk size returns the whole array as one chunk; a missing
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
index 5ec2a39fbd..d1dce67b86 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
@@ -549,4 +549,73 @@ class JsonataEvaluatorTest {
JsonNode result = evaluator.evaluate("{% $map(['{\"a\":1}', '{\"a\":2}'], $parse).a %}", statesVar);
assertEquals("[1,2]", result.toString());
}
+
+ @Test
+ void string_writesAWholeNumberInFullBelowTheExponentBoundary() {
+ // dashjoin stops writing digits at Long.MAX_VALUE, 9.223372036854776E18, and prints
+ // exponent notation from there. AWS keeps writing them until 1e21.
+ JsonNode statesVar = objectMapper.createObjectNode();
+ assertEquals("10000000000000000000", evaluator.evaluate("{% $string(1e19) %}", statesVar).asText());
+ assertEquals("100000000000000000000", evaluator.evaluate("{% $string(1e20) %}", statesVar).asText());
+ assertEquals("150000000000000000000", evaluator.evaluate("{% $string(1.5e20) %}", statesVar).asText());
+ // The largest double under the boundary.
+ assertEquals("999999999999999900000",
+ evaluator.evaluate("{% $string(999999999999999868928) %}", statesVar).asText());
+ }
+
+ @Test
+ void string_switchesToExponentNotationAt1e21() {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ assertEquals("1e+21", evaluator.evaluate("{% $string(1e21) %}", statesVar).asText());
+ // 1e21 - 1 is not a double, so it is the same number as 1e21 and prints the same way.
+ assertEquals("1e+21", evaluator.evaluate("{% $string(1e21 - 1) %}", statesVar).asText());
+ assertEquals("1e+22", evaluator.evaluate("{% $string(1e22) %}", statesVar).asText());
+ assertEquals("1e+100", evaluator.evaluate("{% $string(1e100) %}", statesVar).asText());
+ }
+
+ @Test
+ void string_flipsAtTheSameBoundaryOnANegativeNumber() {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ assertEquals("-100000000000000000000", evaluator.evaluate("{% $string(-1e20) %}", statesVar).asText());
+ assertEquals("-150000000000000000000", evaluator.evaluate("{% $string(-1.5e20) %}", statesVar).asText());
+ assertEquals("-1e+21", evaluator.evaluate("{% $string(-1e21) %}", statesVar).asText());
+ }
+
+ @Test
+ void string_writesTheShortestDigitsThatReadBackAsTheSameDouble() {
+ // The two sides of $parse's number model print differently across the long boundary: the
+ // long prints its exact value, the double prints the shortest decimal that reads back as
+ // itself, so 2^63 is 9223372036854776000 and not its exact 9223372036854775808.
+ JsonNode statesVar = objectMapper.createObjectNode();
+ assertEquals("9223372036854775807",
+ evaluator.evaluate("{% $string($parse('{\"n\":9223372036854775807}').n) %}", statesVar).asText());
+ assertEquals("9223372036854776000",
+ evaluator.evaluate("{% $string($parse('{\"n\":9223372036854775808}').n) %}", statesVar).asText());
+ }
+
+ @Test
+ void string_writesANumberInsideAnObjectOrAnArrayTheSameWay() {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ assertEquals("{\"n\":100000000000000000000}",
+ evaluator.evaluate("{% $string({'n': 1e20}) %}", statesVar).asText());
+ assertEquals("[100000000000000000000,2]",
+ evaluator.evaluate("{% $string([1e20, 2]) %}", statesVar).asText());
+ }
+
+ @Test
+ void string_leavesEveryOtherValueToTheBuiltIn() {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ // AWS switches to exponent notation on the small side too, under 1e-6, and keeps fifteen
+ // significant digits on a fractional number. Both already match.
+ assertEquals("0.000001", evaluator.evaluate("{% $string(1e-6) %}", statesVar).asText());
+ assertEquals("1e-7", evaluator.evaluate("{% $string(1e-7) %}", statesVar).asText());
+ assertEquals("0.333333333333333", evaluator.evaluate("{% $string(1/3) %}", statesVar).asText());
+ assertEquals("true", evaluator.evaluate("{% $string(true) %}", statesVar).asText());
+ assertEquals("abc", evaluator.evaluate("{% $string('abc') %}", statesVar).asText());
+ assertEquals("null", evaluator.evaluate("{% $string(null) %}", statesVar).asText());
+ assertEquals("{\n \"a\": 1\n}", evaluator.evaluate("{% $string({'a': 1}, true) %}", statesVar).asText());
+ assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isNull());
+ assertEquals("[\"100000000000000000000\",\"1e+21\"]",
+ evaluator.evaluate("{% $map([1e20, 1e21], $string) %}", statesVar).toString());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index e53064fd6a..949fc2b1b4 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -52,6 +52,34 @@ class StepFunctionsJsonataIntegrationTest {
assertTrue(output.contains("42"));
}
+ @Test
+ void passStateWithJsonataStringOfALargeWholeNumber() throws Exception {
+ // $string of an id feeds a MessageGroupId, an S3 key or a DynamoDB key, so exponent
+ // notation writes a different key without failing the execution.
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Stringify",
+ "States": {
+ "Stringify": {
+ "Type": "Pass",
+ "Output": {
+ "underTheBoundary": "{% $string(1e20) %}",
+ "atTheBoundary": "{% $string(1e21) %}"
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("jsonata-string-large-whole-number", definition);
+ JsonNode output = objectMapper.readTree(waitForExecution(startExecution(smArn, "{}")));
+
+ assertEquals("100000000000000000000", output.get("underTheBoundary").asText());
+ assertEquals("1e+21", output.get("atTheBoundary").asText());
+ }
+
@Test
void choiceStateWithJsonataCondition() throws Exception {
// Choice state using JSONata Condition instead of Variable/StringEquals