Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(stepfunctions): keep a JSONata expression's explicit null in the output (#2684)

Facundo Paredes48e0ecd80902
diff --git a/docs/services/step-functions.md b/docs/services/step-functions.md
index 2e3d78716f..ee4152a123 100644
--- a/docs/services/step-functions.md
+++ b/docs/services/step-functions.md
@@ -66,6 +66,25 @@ uniformly between zero and the computed delay, as on AWS. One deviation. The del
between attempts is capped at 30 seconds, the same cap Floci applies to `Wait` states,
so emulated runs stay fast.
+## JSONata nulls
+
+An expression that evaluates to JSON `null` produces a value, not a missing one. It keeps its key
+in `Output`, in `Assign` and in a Task's `Arguments`, at any nesting depth, and it stays in place as
+an array element:
+
+```
+"Output": {"v": "{% $states.input.bar %}"} on input {"bar": null} -> {"v": null}
+```
+
+The same `null` is a value inside the expression too: `$exists()` on it is true and `$type()` on it
+is `"null"`. Only an expression that returns nothing loses its key, which is what
+`$states.input.absent` and the functions listed below that evaluate to undefined do.
+
+Two deviations. `$count()` on a JSON null answers `0`; AWS answers `1`. And dropping the key of an
+expression that returned nothing is Floci's own behaviour: AWS fails the state instead, with
+`States.QueryEvaluationError` and `The JSONata expression '$states.input.absent' specified for the
+field 'Output/v' returned nothing (undefined).`
+
## JSONata functions
State machines with `"QueryLanguage": "JSONata"` reach the six functions Step Functions adds on
diff --git a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
index 388fe98a5d..bfa2f02e72 100644
--- a/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
+++ b/src/main/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluator.java
@@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.MissingNode;
import com.fasterxml.jackson.databind.node.NullNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.enterprise.context.ApplicationScoped;
@@ -162,6 +163,11 @@ public class JsonataEvaluator {
String expr = isExpression(expression) ? unwrap(expression) : expression;
try {
Jsonata jsonataExpr = jsonata(expr);
+ // Off, the library keeps JSONata's null marker in the result instead of flattening it
+ // into a Java null. On, an expression that evaluated to JSON null and one that returned
+ // nothing both arrive here as a Java null, and AWS keeps the first while dropping the
+ // second.
+ jsonataExpr.setOutputConvertNulls(false);
Jsonata.Frame frame = jsonataExpr.createFrame();
// Without this the library evaluates unbounded: a recursive expression with no base
// case loops for ever and the execution never leaves RUNNING. The bounds raise a
@@ -261,9 +267,18 @@ public class JsonataEvaluator {
}
JsonNode resolveTemplate(JsonNode template, JsonNode statesVar, JsonNode variables) {
- if (template == null || template.isNull() || template.isMissingNode()) {
+ if (template == null || template.isMissingNode()) {
return template;
}
+ JsonNode resolved = resolveTemplateNode(template, statesVar, variables);
+ // A resolved Output or Arguments is serialized as it stands, and a missing node writes
+ // itself as the empty string, which no client can parse. "Returned nothing" is a value only
+ // in the positions resolveTemplateNode reads it in: an object field it drops, an array
+ // element it fails the state on. As the whole field there is nothing to drop it from.
+ return resolved.isMissingNode() ? NullNode.getInstance() : resolved;
+ }
+
+ private JsonNode resolveTemplateNode(JsonNode template, JsonNode statesVar, JsonNode variables) {
if (template.isTextual()) {
String text = template.asText();
if (isExpression(text)) {
@@ -276,10 +291,11 @@ public class JsonataEvaluator {
Iterator<Map.Entry<String, JsonNode>> fields = template.fields();
while (fields.hasNext()) {
Map.Entry<String, JsonNode> entry = fields.next();
- JsonNode value = resolveTemplate(entry.getValue(), statesVar, variables);
- // Per JSONata spec: undefined (null) values are omitted from object output,
- // matching real AWS Step Functions behavior.
- if (value != null && !value.isNull() && !value.isMissingNode()) {
+ JsonNode value = resolveTemplateNode(entry.getValue(), statesVar, variables);
+ // Per JSONata spec: an expression that returned nothing is omitted from object
+ // output, matching real AWS Step Functions behavior. A JSON null is a value and
+ // keeps its key: AWS answers {"v":null} to Output {"v":"{% null %}"}.
+ if (!value.isMissingNode()) {
resolved.set(entry.getKey(), value);
}
}
@@ -289,10 +305,11 @@ public class JsonataEvaluator {
ArrayNode resolved = objectMapper.createArrayNode();
for (int i = 0; i < template.size(); i++) {
JsonNode element = template.get(i);
- JsonNode value = resolveTemplate(element, statesVar, variables);
- // Per real AWS behavior: undefined array elements fail the execution.
+ JsonNode value = resolveTemplateNode(element, statesVar, variables);
+ // Per real AWS behavior: an array element that returned nothing fails the execution.
// Unlike object fields (which are omitted), undefined in an array is a runtime error.
- if (value == null || value.isNull() || value.isMissingNode()) {
+ // A JSON null is a value and stays: AWS answers [null,1] to ["{% null %}",1].
+ if (value.isMissingNode()) {
String expr = element.isTextual() ? element.asText() : element.toString();
throw new FailStateException("States.Runtime",
"The JSONata expression '" + expr + "' at array index " + i + " returned nothing (undefined).");
@@ -301,21 +318,46 @@ public class JsonataEvaluator {
}
return resolved;
}
- // Primitives (number, boolean) pass through
+ // Primitives (number, boolean, null) pass through
return template;
}
- private Object toObject(JsonNode node) {
- if (node == null || node.isNull() || node.isMissingNode()) {
+ /**
+ * Binds $states and the workflow variables as JSONata values, so a JSON null inside them stays
+ * a null the expression can see: $exists() on it is true and $type() on it is "null", as on
+ * AWS. Only an absent node is undefined.
+ */
+ private static Object toObject(JsonNode node) {
+ if (node == null || node.isMissingNode()) {
return null;
}
- return objectMapper.convertValue(node, Object.class);
+ return toJsonataValue(node);
}
+ /**
+ * The evaluation result. A Java null is the expression returning nothing, which
+ * {@link #resolveTemplateNode} drops from an object; the JSONata null marker is a JSON null,
+ * which it keeps. Nested inside an object or an array there is no undefined, so every null
+ * there is a JSON null.
+ */
private JsonNode toJsonNode(Object value) {
- if (value == null) {
+ return value == null ? MissingNode.getInstance() : fromJsonataValue(value);
+ }
+
+ private JsonNode fromJsonataValue(Object value) {
+ if (value == null || value == Jsonata.NULL_VALUE) {
return NullNode.getInstance();
}
+ if (value instanceof Map<?, ?> map) {
+ ObjectNode object = objectMapper.createObjectNode();
+ map.forEach((key, element) -> object.set(String.valueOf(key), fromJsonataValue(element)));
+ return object;
+ }
+ if (value instanceof List<?> list) {
+ ArrayNode array = objectMapper.createArrayNode();
+ list.forEach(element -> array.add(fromJsonataValue(element)));
+ return array;
+ }
return objectMapper.valueToTree(value);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java
index aa5e008090..f4507b73a2 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/AslExecutorHttpInvokeTest.java
@@ -284,6 +284,47 @@ class AslExecutorHttpInvokeTest {
assertTrue(body.path("active").asBoolean());
}
+ @Test
+ void jsonataArgumentsSendAnExplicitNullAndOmitOnlyWhatReturnedNothing() throws Exception {
+ // AWS keeps the null in the arguments a Task is invoked with: TestState TRACE reports
+ // afterArguments {"FunctionName":"nope","Payload":{"v":null}} for Payload {"v":"{% null %}"}.
+ Execution execution = run("""
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "CallHttp",
+ "States": {
+ "CallHttp": {
+ "Type": "Task",
+ "Resource": "arn:aws:states:::http:invoke",
+ "Arguments": {
+ "ApiEndpoint": "%s/text",
+ "Method": "POST",
+ "Authentication": {
+ "ConnectionArn": "%s"
+ },
+ "RequestBody": {
+ "fromInput": "{%% $lookup($states.input, 'customerId') %%}",
+ "returnedNothing": "{%% $states.input.absent %%}",
+ "active": true
+ }
+ },
+ "End": true
+ }
+ }
+ }
+ """.formatted(baseUrl, CONNECTION_ARN), """
+ {
+ "customerId": null
+ }
+ """);
+
+ assertEquals("SUCCEEDED", execution.getStatus(), execution.getCause());
+ JsonNode body = objectMapper.readTree(onlyRequest().body());
+ assertTrue(body.path("fromInput").isNull(), body.toString());
+ assertTrue(body.path("returnedNothing").isMissingNode(), body.toString());
+ assertTrue(body.path("active").asBoolean());
+ }
+
@Test
void nonSuccessStatusFailsWithStatesHttpStatusCodeError() throws Exception {
Execution execution = run("""
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
index e9d09fe268..a953eaa0cf 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/JsonataEvaluatorTest.java
@@ -85,12 +85,12 @@ class JsonataEvaluatorTest {
}
@Test
- void evaluate_returnsNullForMissingField() throws Exception {
+ void evaluate_returnsNothingForMissingField() throws Exception {
JsonNode statesVar = objectMapper.readTree("""
{"input": {"name": "Alice"}}
""");
JsonNode result = evaluator.evaluate("{% $states.input.missing %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
@@ -216,7 +216,7 @@ class JsonataEvaluatorTest {
void parse_noArgumentReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $parse() %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
@@ -260,14 +260,14 @@ class JsonataEvaluatorTest {
void partition_emptyArrayReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $partition([], 2) %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
void partition_chunkSizeZeroReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $partition([1,2,3], 0) %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
@@ -319,21 +319,21 @@ class JsonataEvaluatorTest {
void range_wrongSignStepReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $range(5, 1, 1) %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
void range_stepZeroReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $range(1, 5, 0) %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
void range_missingStepReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $range(1, 5) %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
@@ -375,7 +375,7 @@ class JsonataEvaluatorTest {
void hash_missingAlgorithmReturnsUndefined() {
JsonNode statesVar = objectMapper.createObjectNode();
JsonNode result = evaluator.evaluate("{% $hash('Hello') %}", statesVar);
- assertTrue(result.isNull());
+ assertTrue(result.isMissingNode());
}
@Test
@@ -450,7 +450,7 @@ class JsonataEvaluatorTest {
assertEquals("[-2,-1,0]", evaluator.evaluate("{% $range(-2.9, 0, 1) %}", statesVar).toString());
assertEquals("[5,4,3,2,1]", evaluator.evaluate("{% $range(5, 1, -1.5) %}", statesVar).toString());
// -0.5 rounds to 0, which is undefined, while -2 stays negative and is an error.
- assertTrue(evaluator.evaluate("{% $partition([1,2,3,4], -0.5) %}", statesVar).isNull());
+ assertTrue(evaluator.evaluate("{% $partition([1,2,3,4], -0.5) %}", statesVar).isMissingNode());
}
@Test
@@ -480,7 +480,7 @@ class JsonataEvaluatorTest {
@Test
void hash_missingAlgorithmIsUndefinedButNonStringOneIsASignatureError() {
JsonNode statesVar = objectMapper.createObjectNode();
- assertTrue(evaluator.evaluate("{% $hash('a') %}", statesVar).isNull());
+ assertTrue(evaluator.evaluate("{% $hash('a') %}", statesVar).isMissingNode());
AslExecutor.FailStateException ex = assertThrows(AslExecutor.FailStateException.class,
() -> evaluator.evaluate("{% $hash('a', 5) %}", statesVar));
assertTrue(ex.cause.contains("T0410"));
@@ -683,7 +683,9 @@ class JsonataEvaluatorTest {
assertEquals("abc", evaluator.evaluate("{% $string('abc') %}", statesVar).asText());
assertEquals("null", evaluator.evaluate("{% $string(null) %}", statesVar).asText());
assertEquals("{\n \"a\": 1\n}", evaluator.evaluate("{% $string({'a': 1}, true) %}", statesVar).asText());
- assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isNull());
+ // $string() with no argument returns nothing, which this change stops reporting as an
+ // explicit null. AWS fails the state on it, which is #2665's subject and not this one's.
+ assertTrue(evaluator.evaluate("{% $string() %}", statesVar).isMissingNode());
assertEquals("[\"100000000000000000000\",\"1e+21\"]",
evaluator.evaluate("{% $map([1e20, 1e21], $string) %}", statesVar).toString());
}
@@ -800,4 +802,87 @@ class JsonataEvaluatorTest {
assertEquals("States.QueryEvaluationError", ex.error);
assertTrue(ex.cause.contains("nope"), ex.cause);
}
+
+ @Test
+ void anExplicitNullKeepsItsKeyAndOnlyWhatReturnedNothingIsDropped() throws Exception {
+ JsonNode statesVar = objectMapper.readTree("""
+ {"input": {"bar": null}}
+ """);
+ JsonNode template = objectMapper.readTree("""
+ {"fromInput": "{% $lookup($states.input, 'bar') %}",
+ "returnedNothing": "{% $states.input.absent %}",
+ "kept": 1}
+ """);
+
+ assertEquals("{\"fromInput\":null,\"kept\":1}",
+ evaluator.resolveTemplate(template, statesVar).toString());
+ }
+
+ @Test
+ void anExplicitNullSurvivesInsideANestedObject() throws Exception {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ JsonNode template = objectMapper.readTree("""
+ {"outer": {"fromExpression": "{% null %}", "kept": 1}}
+ """);
+
+ assertEquals("{\"outer\":{\"fromExpression\":null,\"kept\":1}}",
+ evaluator.resolveTemplate(template, statesVar).toString());
+ }
+
+ @Test
+ void aWholeTemplateIsANullWhetherItEvaluatedToNullOrReturnedNothing() throws Exception {
+ JsonNode statesVar = objectMapper.createObjectNode();
+
+ assertTrue(evaluator.resolveTemplate(objectMapper.readTree("\"{% null %}\""), statesVar).isNull());
+ // AWS fails the second one with States.QueryEvaluationError, which #2665 covers. Until then
+ // it is a null and not a missing node: an Output or Arguments is serialized as it stands,
+ // and a missing node writes itself as the empty string rather than as JSON.
+ assertTrue(evaluator.resolveTemplate(objectMapper.readTree("\"{% $states.input.absent %}\""), statesVar)
+ .isNull());
+ }
+
+ @Test
+ void anExplicitNullIsAnArrayElementWhileNothingStillFailsTheState() throws Exception {
+ JsonNode statesVar = objectMapper.createObjectNode();
+
+ assertEquals("{\"values\":[null,1]}", evaluator.resolveTemplate(objectMapper.readTree("""
+ {"values": ["{% null %}", 1]}
+ """), statesVar).toString());
+
+ AslExecutor.FailStateException failure = assertThrows(AslExecutor.FailStateException.class,
+ () -> evaluator.resolveTemplate(objectMapper.readTree("""
+ {"values": ["{% $states.input.absent %}"]}
+ """), statesVar));
+ assertTrue(failure.cause.contains("returned nothing (undefined)"), failure.cause);
+ }
+
+ @Test
+ void aLiteralNullInTheTemplateKeepsItsKey() throws Exception {
+ JsonNode statesVar = objectMapper.createObjectNode();
+
+ assertEquals("{\"literal\":null,\"kept\":1}", evaluator.resolveTemplate(objectMapper.readTree("""
+ {"literal": null, "kept": 1}
+ """), statesVar).toString());
+ }
+
+ @Test
+ void anInputNullIsAValueForExistsAndType() throws Exception {
+ JsonNode statesVar = objectMapper.readTree("""
+ {"input": {"bar": null}}
+ """);
+
+ assertTrue(evaluator.evaluate("{% $exists($states.input.bar) %}", statesVar).asBoolean());
+ assertEquals("null", evaluator.evaluate("{% $type($states.input.bar) %}", statesVar).asText());
+ }
+
+ @Test
+ void anAssignedNullReadsBackAsANull() throws Exception {
+ JsonNode statesVar = objectMapper.createObjectNode();
+ JsonNode variables = objectMapper.readTree("""
+ {"nullVariable": null}
+ """);
+
+ assertTrue(evaluator.evaluate("{% $nullVariable %}", statesVar, variables).isNull());
+ assertTrue(evaluator.evaluate("{% $exists($nullVariable) %}", statesVar, variables).asBoolean());
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
index 949fc2b1b4..9587bd2a59 100644
--- a/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/stepfunctions/StepFunctionsJsonataIntegrationTest.java
@@ -1225,6 +1225,104 @@ class StepFunctionsJsonataIntegrationTest {
assertTrue(output.contains("Jane Doe"));
}
+ @Test
+ void outputKeepsAnExplicitNullInEveryPositionAndDropsOnlyWhatReturnedNothing() throws Exception {
+ // AWS returns {"v":null} for an expression that evaluates to JSON null, in an object field,
+ // nested, as an array element and as a literal. Only an expression that returned nothing
+ // loses its key.
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Transform",
+ "States": {
+ "Transform": {
+ "Type": "Pass",
+ "Output": {
+ "fromInput": "{% $lookup($states.input, 'bar') %}",
+ "fromLiteralExpression": "{% null %}",
+ "literal": null,
+ "nested": {"inner": "{% null %}", "kept": 1},
+ "values": ["{% null %}", 1],
+ "returnedNothing": "{% $states.input.absent %}"
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("jsonata-explicit-null-test", definition);
+ String execArn = startExecution(smArn, "{\"bar\": null}");
+ JsonNode output = objectMapper.readTree(waitForExecution(execArn));
+
+ assertTrue(output.path("fromInput").isNull(), output.toString());
+ assertTrue(output.path("fromLiteralExpression").isNull(), output.toString());
+ assertTrue(output.path("literal").isNull(), output.toString());
+ assertTrue(output.path("nested").path("inner").isNull(), output.toString());
+ assertEquals("[null,1]", output.path("values").toString());
+ assertTrue(output.path("returnedNothing").isMissingNode(), output.toString());
+ }
+
+ @Test
+ void anAssignedNullReadsBackAsANullInTheNextState() throws Exception {
+ // AWS keeps the null in the variables map: TestState TRACE reports {"x":null} for
+ // Assign {"x": "{% null %}"}. A state never sees its own Assign, so the read is one state on.
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "AssignNull",
+ "States": {
+ "AssignNull": {
+ "Type": "Pass",
+ "Assign": {"nullVariable": "{% $states.input.bar %}"},
+ "Next": "ReadItBack"
+ },
+ "ReadItBack": {
+ "Type": "Pass",
+ "Output": {
+ "fromVariable": "{% $nullVariable %}",
+ "exists": "{% $exists($nullVariable) %}"
+ },
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("jsonata-assign-null-test", definition);
+ String execArn = startExecution(smArn, "{\"bar\": null}");
+ JsonNode output = objectMapper.readTree(waitForExecution(execArn));
+
+ assertTrue(output.path("fromVariable").isNull(), output.toString());
+ assertTrue(output.path("exists").asBoolean(), output.toString());
+ }
+
+ @Test
+ void aWholeOutputThatReturnedNothingStaysReadableJson() throws Exception {
+ // AWS fails this state with States.QueryEvaluationError, which #2665 covers. Until then the
+ // execution succeeds, and what DescribeExecution hands back has to parse: an output field
+ // holding the empty string is not JSON any client can read.
+ String definition = """
+ {
+ "QueryLanguage": "JSONata",
+ "StartAt": "Transform",
+ "States": {
+ "Transform": {
+ "Type": "Pass",
+ "Output": "{% $states.input.absent %}",
+ "End": true
+ }
+ }
+ }
+ """;
+
+ String smArn = createStateMachine("jsonata-whole-output-nothing-test", definition);
+ String execArn = startExecution(smArn, "{}");
+ String output = waitForExecution(execArn);
+
+ assertTrue(objectMapper.readTree(output).isNull(), output);
+ }
+
@Test
void assignedVariablesSurviveBeyondNextStateOutput() throws Exception {
// Variables set via Assign persist across states even after a later state replaces the output.