Light, fluffy, and always free - The AWS Local Emulator alternative
fix(stepfunctions): keep a JSONata expression's explicit null in the output (#2684)
Facundo Paredes48e0ecd80902
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.mdindex 2e3d78716f..ee4152a123 100644--- a/docs/services/step-functions.md+++ b/docs/services/step-functions.md@@ -66,6 +66,25 @@ uniformly between zero and the computed delay, as on AWS. One deviation. The delbetween attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,so emulated runs stay fast.+## JSONata nulls++An expression that evaluates to JSON `null` produces a value, not a missing one. It keeps its key+in `Output`, in `Assign` and in a Task's `Arguments`, at any nesting depth, and it stays in place as+an array element:++```+"Output": {"v": "{% $states.input.bar %}"} on input {"bar": null} -> {"v": null}+```++The same `null` is a value inside the expression too: `$exists()` on it is true and `$type()` on it+is `"null"`. Only an expression that returns nothing loses its key, which is what+`$states.input.absent` and the functions listed below that evaluate to undefined do.++Two deviations. `$count()` on a JSON null answers `0`; AWS answers `1`. And dropping the key of an+expression that returned nothing is Floci's own behaviour: AWS fails the state instead, with+`States.QueryEvaluationError` and `The JSONata expression '$states.input.absent' specified for the+field 'Output/v' returned nothing (undefined).`+## JSONata functionsState machines with `"QueryLanguage": "JSONata"` reach the six functions Step Functions adds ondiff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.javaindex 388fe98a5d..bfa2f02e72 100644--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java@@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.ObjectReader;import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.MissingNode;import com.fasterxml.jackson.databind.node.NullNode;import com.fasterxml.jackson.databind.node.ObjectNode;import jakarta.enterprise.context.ApplicationScoped;@@ -162,6 +163,11 @@ public class JsonataEvaluator {String expr = isExpression(expression) ? unwrap(expression) : expression;try {Jsonata jsonataExpr = jsonata(expr);+ // Off, the library keeps JSONata's null marker in the result instead of flattening it+ // into a Java null. On, an expression that evaluated to JSON null and one that returned+ // nothing both arrive here as a Java null, and AWS keeps the first while dropping the+ // second.+ jsonataExpr.setOutputConvertNulls(false);Jsonata.Frame frame = jsonataExpr.createFrame();// Without this the library evaluates unbounded: a recursive expression with no base// case loops for ever and the execution never leaves RUNNING. The bounds raise a@@ -261,9 +267,18 @@ public class JsonataEvaluator {}JsonNode resolveTemplate(JsonNode template, JsonNode statesVar, JsonNode variables) {- if (template == null || template.isNull() || template.isMissingNode()) {+ if (template == null || template.isMissingNode()) {return template;}+ JsonNode resolved = resolveTemplateNode(template, statesVar, variables);+ // A resolved Output or Arguments is serialized as it stands, and a missing node writes+ // itself as the empty string, which no client can parse. "Returned nothing" is a value only+ // in the positions resolveTemplateNode reads it in: an object field it drops, an array+ // element it fails the state on. As the whole field there is nothing to drop it from.+ return resolved.isMissingNode() ? NullNode.getInstance() : resolved;+ }++ private JsonNode resolveTemplateNode(JsonNode template, JsonNode statesVar, JsonNode variables) {if (template.isTextual()) {String text = template.asText();if (isExpression(text)) {@@ -276,10 +291,11 @@ public class JsonataEvaluator {Iterator<Map.Entry<String, JsonNode>> fields = template.fields();while (fields.hasNext()) {Map.Entry<String, JsonNode> entry = fields.next();- JsonNode value = resolveTemplate(entry.getValue(), statesVar, variables);- // Per JSONata spec: undefined (null) values are omitted from object output,- // matching real AWS Step Functions behavior.- if (value != null && !value.isNull() && !value.isMissingNode()) {+ JsonNode value = resolveTemplateNode(entry.getValue(), statesVar, variables);+ // Per JSONata spec: an expression that returned nothing is omitted from object+ // output, matching real AWS Step Functions behavior. A JSON null is a value and+ // keeps its key: AWS answers {"v":null} to Output {"v":"{% null %}"}.+ if (!value.isMissingNode()) {resolved.set(entry.getKey(), value);}}@@ -289,10 +305,11 @@ public class JsonataEvaluator {ArrayNode resolved = objectMapper.createArrayNode();for (int i = 0; i < template.size(); i++) {JsonNode element = template.get(i);- JsonNode value = resolveTemplate(element, statesVar, variables);- // Per real AWS behavior: undefined array elements fail the execution.+ JsonNode value = resolveTemplateNode(element, statesVar, variables);+ // Per real AWS behavior: an array element that returned nothing fails the execution.// Unlike object fields (which are omitted), undefined in an array is a runtime error.- if (value == null || value.isNull() || value.isMissingNode()) {+ // A JSON null is a value and stays: AWS answers [null,1] to ["{% null %}",1].+ if (value.isMissingNode()) {String expr = element.isTextual() ? element.asText() : element.toString();throw new FailStateException("States.Runtime","The JSONata expression '" + expr + "' at array index " + i + " returned nothing (undefined).");@@ -301,21 +318,46 @@ public class JsonataEvaluator {}return resolved;}- // Primitives (number, boolean) pass through+ // Primitives (number, boolean, null) pass throughreturn template;}- private Object toObject(JsonNode node) {- if (node == null || node.isNull() || node.isMissingNode()) {+ /**+ * Binds $states and the workflow variables as JSONata values, so a JSON null inside them stays+ * a null the expression can see: $exists() on it is true and $type() on it is "null", as on+ * AWS. Only an absent node is undefined.+ */+ private static Object toObject(JsonNode node) {+ if (node == null || node.isMissingNode()) {return null;}- return objectMapper.convertValue(node, Object.class);+ return toJsonataValue(node);}+ /**+ * The evaluation result. A Java null is the expression returning nothing, which+ * {@link #resolveTemplateNode} drops from an object; the JSONata null marker is a JSON null,+ * which it keeps. Nested inside an object or an array there is no undefined, so every null+ * there is a JSON null.+ */private JsonNode toJsonNode(Object value) {- if (value == null) {+ return value == null ? MissingNode.getInstance() : fromJsonataValue(value);+ }++ private JsonNode fromJsonataValue(Object value) {+ if (value == null || value == Jsonata.NULL_VALUE) {return NullNode.getInstance();}+ if (value instanceof Map<?, ?> map) {+ ObjectNode object = objectMapper.createObjectNode();+ map.forEach((key, element) -> object.set(String.valueOf(key), fromJsonataValue(element)));+ return object;+ }+ if (value instanceof List<?> list) {+ ArrayNode array = objectMapper.createArrayNode();+ list.forEach(element -> array.add(fromJsonataValue(element)));+ return array;+ }return objectMapper.valueToTree(value);}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.javaindex aa5e008090..f4507b73a2 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java@@ -284,6 +284,47 @@ class AslExecutorHttpInvokeTest {assertTrue(body.path("active").asBoolean());}+ @Test+ void jsonataArgumentsSendAnExplicitNullAndOmitOnlyWhatReturnedNothing() throws Exception {+ // AWS keeps the null in the arguments a Task is invoked with: TestState TRACE reports+ // afterArguments {"FunctionName":"nope","Payload":{"v":null}} for Payload {"v":"{% null %}"}.+ Execution execution = run("""+ {+ "QueryLanguage": "JSONata",+ "StartAt": "CallHttp",+ "States": {+ "CallHttp": {+ "Type": "Task",+ "Resource": "arn:aws:states:::http:invoke",+ "Arguments": {+ "ApiEndpoint": "%s/text",+ "Method": "POST",+ "Authentication": {+ "ConnectionArn": "%s"+ },+ "RequestBody": {+ "fromInput": "{%% $lookup($states.input, 'customerId') %%}",+ "returnedNothing": "{%% $states.input.absent %%}",+ "active": true+ }+ },+ "End": true+ }+ }+ }+ """.formatted(baseUrl, CONNECTION_ARN), """+ {+ "customerId": null+ }+ """);++ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());+ JsonNode body = objectMapper.readTree(onlyRequest().body());+ assertTrue(body.path("fromInput").isNull(), body.toString());+ assertTrue(body.path("returnedNothing").isMissingNode(), body.toString());+ assertTrue(body.path("active").asBoolean());+ }+@Testvoid nonSuccessStatusFailsWithStatesHttpStatusCodeError() throws Exception {Execution execution = run("""diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.javaindex e9d09fe268..a953eaa0cf 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java@@ -85,12 +85,12 @@ class JsonataEvaluatorTest {}@Test- void evaluate_returnsNullForMissingField() throws Exception {+ void evaluate_returnsNothingForMissingField() throws Exception {JsonNode statesVar = objectMapper.readTree("""{"input": {"name": "Alice"}}""");JsonNode result = evaluator.evaluate("{% $states.input.missing %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Test@@ -216,7 +216,7 @@ class JsonataEvaluatorTest {void parse_noArgumentReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $parse() %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Test@@ -260,14 +260,14 @@ class JsonataEvaluatorTest {void partition_emptyArrayReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $partition([], 2) %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Testvoid partition_chunkSizeZeroReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $partition([1,2,3], 0) %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Test@@ -319,21 +319,21 @@ class JsonataEvaluatorTest {void range_wrongSignStepReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $range(5, 1, 1) %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Testvoid range_stepZeroReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $range(1, 5, 0) %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Testvoid range_missingStepReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $range(1, 5) %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Test@@ -375,7 +375,7 @@ class JsonataEvaluatorTest {void hash_missingAlgorithmReturnsUndefined() {JsonNode statesVar = objectMapper.createObjectNode();JsonNode result = evaluator.evaluate("{% $hash('Hello') %}", statesVar);- assertTrue(result.isNull());+ assertTrue(result.isMissingNode());}@Test@@ -450,7 +450,7 @@ class JsonataEvaluatorTest {assertEquals("[-2,-1,0]", evaluator.evaluate("{% $range(-2.9, 0, 1) %}", statesVar).toString());assertEquals("[5,4,3,2,1]", evaluator.evaluate("{% $range(5, 1, -1.5) %}", statesVar).toString());// -0.5 rounds to 0, which is undefined, while -2 stays negative and is an error.- assertTrue(evaluator.evaluate("{% $partition([1,2,3,4], -0.5) %}", statesVar).isNull());+ assertTrue(evaluator.evaluate("{% $partition([1,2,3,4], -0.5) %}", statesVar).isMissingNode());}@Test@@ -480,7 +480,7 @@ class JsonataEvaluatorTest {@Testvoid hash_missingAlgorithmIsUndefinedButNonStringOneIsASignatureError() {JsonNode statesVar = objectMapper.createObjectNode();- assertTrue(evaluator.evaluate("{% $hash('a') %}", statesVar).isNull());+ assertTrue(evaluator.evaluate("{% $hash('a') %}", statesVar).isMissingNode());AslExecutor.FailStateException ex = assertThrows(AslExecutor.FailStateException.class,() -> evaluator.evaluate("{% $hash('a', 5) %}", statesVar));assertTrue(ex.cause.contains("T0410"));@@ -683,7 +683,9 @@ class JsonataEvaluatorTest {assertEquals("abc", evaluator.evaluate("{% $string('abc') %}", statesVar).asText());assertEquals("null", evaluator.evaluate("{% $string(null) %}", statesVar).asText());assertEquals("{\n \"a\": 1\n}", evaluator.evaluate("{% $string({'a': 1}, true) %}", statesVar).asText());- assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isNull());+ // $string() with no argument returns nothing, which this change stops reporting as an+ // explicit null. AWS fails the state on it, which is #2665's subject and not this one's.+ assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isMissingNode());assertEquals("[\"100000000000000000000\",\"1e+21\"]",evaluator.evaluate("{% $map([1e20, 1e21], $string) %}", statesVar).toString());}@@ -800,4 +802,87 @@ class JsonataEvaluatorTest {assertEquals("States.QueryEvaluationError", ex.error);assertTrue(ex.cause.contains("nope"), ex.cause);}++ @Test+ void anExplicitNullKeepsItsKeyAndOnlyWhatReturnedNothingIsDropped() throws Exception {+ JsonNode statesVar = objectMapper.readTree("""+ {"input": {"bar": null}}+ """);+ JsonNode template = objectMapper.readTree("""+ {"fromInput": "{% $lookup($states.input, 'bar') %}",+ "returnedNothing": "{% $states.input.absent %}",+ "kept": 1}+ """);++ assertEquals("{\"fromInput\":null,\"kept\":1}",+ evaluator.resolveTemplate(template, statesVar).toString());+ }++ @Test+ void anExplicitNullSurvivesInsideANestedObject() throws Exception {+ JsonNode statesVar = objectMapper.createObjectNode();+ JsonNode template = objectMapper.readTree("""+ {"outer": {"fromExpression": "{% null %}", "kept": 1}}+ """);++ assertEquals("{\"outer\":{\"fromExpression\":null,\"kept\":1}}",+ evaluator.resolveTemplate(template, statesVar).toString());+ }++ @Test+ void aWholeTemplateIsANullWhetherItEvaluatedToNullOrReturnedNothing() throws Exception {+ JsonNode statesVar = objectMapper.createObjectNode();++ assertTrue(evaluator.resolveTemplate(objectMapper.readTree("\"{% null %}\""), statesVar).isNull());+ // AWS fails the second one with States.QueryEvaluationError, which #2665 covers. Until then+ // it is a null and not a missing node: an Output or Arguments is serialized as it stands,+ // and a missing node writes itself as the empty string rather than as JSON.+ assertTrue(evaluator.resolveTemplate(objectMapper.readTree("\"{% $states.input.absent %}\""), statesVar)+ .isNull());+ }++ @Test+ void anExplicitNullIsAnArrayElementWhileNothingStillFailsTheState() throws Exception {+ JsonNode statesVar = objectMapper.createObjectNode();++ assertEquals("{\"values\":[null,1]}", evaluator.resolveTemplate(objectMapper.readTree("""+ {"values": ["{% null %}", 1]}+ """), statesVar).toString());++ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,+ () -> evaluator.resolveTemplate(objectMapper.readTree("""+ {"values": ["{% $states.input.absent %}"]}+ """), statesVar));+ assertTrue(failure.cause.contains("returned nothing (undefined)"), failure.cause);+ }++ @Test+ void aLiteralNullInTheTemplateKeepsItsKey() throws Exception {+ JsonNode statesVar = objectMapper.createObjectNode();++ assertEquals("{\"literal\":null,\"kept\":1}", evaluator.resolveTemplate(objectMapper.readTree("""+ {"literal": null, "kept": 1}+ """), statesVar).toString());+ }++ @Test+ void anInputNullIsAValueForExistsAndType() throws Exception {+ JsonNode statesVar = objectMapper.readTree("""+ {"input": {"bar": null}}+ """);++ assertTrue(evaluator.evaluate("{% $exists($states.input.bar) %}", statesVar).asBoolean());+ assertEquals("null", evaluator.evaluate("{% $type($states.input.bar) %}", statesVar).asText());+ }++ @Test+ void anAssignedNullReadsBackAsANull() throws Exception {+ JsonNode statesVar = objectMapper.createObjectNode();+ JsonNode variables = objectMapper.readTree("""+ {"nullVariable": null}+ """);++ assertTrue(evaluator.evaluate("{% $nullVariable %}", statesVar, variables).isNull());+ assertTrue(evaluator.evaluate("{% $exists($nullVariable) %}", statesVar, variables).asBoolean());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.javaindex 949fc2b1b4..9587bd2a59 100644--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java@@ -1225,6 +1225,104 @@ class StepFunctionsJsonataIntegrationTest {assertTrue(output.contains("Jane Doe"));}+ @Test+ void outputKeepsAnExplicitNullInEveryPositionAndDropsOnlyWhatReturnedNothing() throws Exception {+ // AWS returns {"v":null} for an expression that evaluates to JSON null, in an object field,+ // nested, as an array element and as a literal. Only an expression that returned nothing+ // loses its key.+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Transform",+ "States": {+ "Transform": {+ "Type": "Pass",+ "Output": {+ "fromInput": "{% $lookup($states.input, 'bar') %}",+ "fromLiteralExpression": "{% null %}",+ "literal": null,+ "nested": {"inner": "{% null %}", "kept": 1},+ "values": ["{% null %}", 1],+ "returnedNothing": "{% $states.input.absent %}"+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("jsonata-explicit-null-test", definition);+ String execArn = startExecution(smArn, "{\"bar\": null}");+ JsonNode output = objectMapper.readTree(waitForExecution(execArn));++ assertTrue(output.path("fromInput").isNull(), output.toString());+ assertTrue(output.path("fromLiteralExpression").isNull(), output.toString());+ assertTrue(output.path("literal").isNull(), output.toString());+ assertTrue(output.path("nested").path("inner").isNull(), output.toString());+ assertEquals("[null,1]", output.path("values").toString());+ assertTrue(output.path("returnedNothing").isMissingNode(), output.toString());+ }++ @Test+ void anAssignedNullReadsBackAsANullInTheNextState() throws Exception {+ // AWS keeps the null in the variables map: TestState TRACE reports {"x":null} for+ // Assign {"x": "{% null %}"}. A state never sees its own Assign, so the read is one state on.+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "AssignNull",+ "States": {+ "AssignNull": {+ "Type": "Pass",+ "Assign": {"nullVariable": "{% $states.input.bar %}"},+ "Next": "ReadItBack"+ },+ "ReadItBack": {+ "Type": "Pass",+ "Output": {+ "fromVariable": "{% $nullVariable %}",+ "exists": "{% $exists($nullVariable) %}"+ },+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("jsonata-assign-null-test", definition);+ String execArn = startExecution(smArn, "{\"bar\": null}");+ JsonNode output = objectMapper.readTree(waitForExecution(execArn));++ assertTrue(output.path("fromVariable").isNull(), output.toString());+ assertTrue(output.path("exists").asBoolean(), output.toString());+ }++ @Test+ void aWholeOutputThatReturnedNothingStaysReadableJson() throws Exception {+ // AWS fails this state with States.QueryEvaluationError, which #2665 covers. Until then the+ // execution succeeds, and what DescribeExecution hands back has to parse: an output field+ // holding the empty string is not JSON any client can read.+ String definition = """+ {+ "QueryLanguage": "JSONata",+ "StartAt": "Transform",+ "States": {+ "Transform": {+ "Type": "Pass",+ "Output": "{% $states.input.absent %}",+ "End": true+ }+ }+ }+ """;++ String smArn = createStateMachine("jsonata-whole-output-nothing-test", definition);+ String execArn = startExecution(smArn, "{}");+ String output = waitForExecution(execArn);++ assertTrue(objectMapper.readTree(output).isNull(), output);+ }+@Testvoid assignedVariablesSurviveBeyondNextStateOutput() throws Exception {// Variables set via Assign persist across states even after a later state replaces the output.