Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(sfn): evaluate JSONata Fail Error/Cause and Wait Seconds/Timestamp (#2167) (#2203)

Lucas9e3909dfd7e1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fec54677..60874fea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
- **Auth — header signatures are verified for IAM user and STS session keys** — under `AUTH=true` a request signed with such a key and the wrong secret, or changed after signing, was served; it now answers `SignatureDoesNotMatch` (or the service's `InvalidSignatureException`), an `Authorization` header without its `Signature` or `SignedHeaders` answers `IncompleteSignature` (Lambda `MissingAuthenticationTokenException`, S3 `AuthorizationHeaderMalformed`) and a credential date other than the request date `SignatureDoesNotMatch` (S3 `AuthorizationHeaderMalformed`), S3 unsigned, streaming and signed-chunk payloads are checked the same way, and the root key still accepts any secret. Contributed by @iot-rocket.
### Fixed
+- **Step Functions — JSONata Fail and Wait fields are evaluated** — a JSONata Fail state raised its `Error` and `Cause` expressions as literal text, a JSONata Wait failed on a `Seconds` expression, ignored a `Timestamp` expression and dropped its `Output` and `Assign`, and JSONPath `ErrorPath`/`CausePath` were ignored; they are now evaluated, with AWS's `States.QueryEvaluationError` or `States.Runtime` for a result of the wrong type, a `Seconds` that is not a non-negative integer and an unparsable `Timestamp`.
- **S3 — presigned URLs keep AWS's time limits under `AUTH=true`** — an `X-Amz-Expires` over 604800 seconds or not a number, or a malformed `X-Amz-Date`, answers 400 `AuthorizationQueryParametersError`, and a URL dated more than 15 minutes ahead answers 403 `AccessDenied` `Request is not yet valid`, where they were served. Contributed by @iot-rocket.
- **Auth — credential failures in the service's protocol** — under `AUTH=true` an unknown, inactive or expired access key answered `UnrecognizedClientException` 403 from every service; it now gets the code, status and `Content-Type` AWS sends for the service's protocol, for example `InvalidClientTokenId` 403 from IAM and SQS, `UnrecognizedClientException` 400 from DynamoDB, `InvalidAccessKeyId` 403 from S3 and `AuthFailure` 401 from EC2. Contributed by @iot-rocket.
### Fixed
diff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.py
index dfcd9f23..f9fb1ea1 100644
--- a/ministack/services/stepfunctions.py
+++ b/ministack/services/stepfunctions.py
@@ -1276,11 +1276,8 @@ def _test_state(data):
result = {"status": "SUCCEEDED", "output": json.dumps(output)}
elif state_type == "Fail":
- result = {
- "status": "FAILED",
- "error": state_def.get("Error", "States.Fail"),
- "cause": state_def.get("Cause", ""),
- }
+ error, cause = _fail_state_error(state_def, input_data, ctx)
+ result = {"status": "FAILED", "error": error, "cause": cause}
elif state_type == "Task":
effective = _apply_input_path(state_def, input_data, ctx)
@@ -1576,9 +1573,7 @@ def _run_execution(exec_arn):
continue
if state_type == "Fail":
- raise _ExecutionError(
- state_def.get("Error", "States.Fail"),
- state_def.get("Cause", ""))
+ raise _ExecutionError(*_fail_state_error(state_def, current_input, ctx))
handler_fn = {
"Pass": _execute_pass,
@@ -2181,6 +2176,17 @@ def _evaluate_rule(rule, data, ctx=None):
# ---------------------------------------------------------------------------
def _execute_wait(state_def, raw_input, ctx=None):
+ if _state_query_language(state_def, ctx) == "JSONata":
+ # JSONata Wait: Seconds and Timestamp may be expressions, and Output
+ # and Assign replace the JSONPath chain, as for Pass.
+ if "Seconds" in state_def:
+ _scaled_sleep(_jsonata_wait_seconds(state_def, raw_input, ctx))
+ elif "Timestamp" in state_def:
+ _sleep_until(_jsonata_wait_timestamp(state_def, raw_input, ctx))
+ output = _apply_jsonata_output(state_def, raw_input, ctx, default=raw_input)
+ _apply_state_assign(state_def, raw_input, ctx)
+ return output, _next_or_end(state_def)
+
effective = _apply_input_path(state_def, raw_input, ctx)
if "Seconds" in state_def:
@@ -2200,6 +2206,29 @@ def _execute_wait(state_def, raw_input, ctx=None):
return output, _next_or_end(state_def)
+def _jsonata_wait_seconds(state_def, raw_input, ctx):
+ seconds = _jsonata_state_field(state_def, "Seconds", "number", raw_input, ctx)
+ if not float(seconds).is_integer():
+ raise _ExecutionError(
+ "States.QueryEvaluationError",
+ f"The Seconds field cannot be parsed as an integer: {json.dumps(seconds)}")
+ if seconds < 0:
+ raise _ExecutionError(
+ "States.QueryEvaluationError",
+ f"Seconds cannot be negative but was: {json.dumps(seconds)}")
+ return seconds
+
+
+def _jsonata_wait_timestamp(state_def, raw_input, ctx):
+ timestamp = _jsonata_state_field(state_def, "Timestamp", "string", raw_input, ctx)
+ if _parse_ts(timestamp) is None:
+ raise _ExecutionError(
+ "States.QueryEvaluationError",
+ "The Timestamp field cannot be parsed as an ISO-8601 extended offset "
+ f"date-time format string: {timestamp}")
+ return timestamp
+
+
def _scaled_sleep(seconds):
scaled = seconds * _SFN_WAIT_SCALE
if scaled > 0:
@@ -2216,6 +2245,34 @@ def _sleep_until(iso_ts):
pass
+# ---------------------------------------------------------------------------
+# Fail state
+# ---------------------------------------------------------------------------
+
+def _fail_state_error(state_def, raw_input, ctx=None):
+ """The error and cause a Fail state ends with. JSONata states may give
+ either as an expression; JSONPath states may select either from the
+ input with ErrorPath or CausePath. Both must be strings."""
+ return (_fail_state_field(state_def, "Error", "States.Fail", raw_input, ctx),
+ _fail_state_field(state_def, "Cause", "", raw_input, ctx))
+
+
+def _fail_state_field(state_def, field, default, raw_input, ctx):
+ if field in state_def and _state_query_language(state_def, ctx) == "JSONata":
+ return _jsonata_state_field(state_def, field, "string", raw_input, ctx)
+ path = state_def.get(f"{field}Path")
+ if path is None:
+ return state_def.get(field, default)
+ value = (_evaluate_intrinsic(path, raw_input, ctx) if path.startswith("States.")
+ else _resolve_path_or_ctx(path, raw_input, ctx))
+ if not isinstance(value, str):
+ raise _ExecutionError(
+ "States.Runtime",
+ f"The value referenced by {field}Path must be a string, "
+ f"but the value was: {json.dumps(value)}")
+ return value
+
+
# ---------------------------------------------------------------------------
# Parallel state
# ---------------------------------------------------------------------------
@@ -2385,9 +2442,7 @@ def _run_sub_machine(states, start_at, input_data, execution, ctx):
return _apply_output_path(state_def,
_apply_input_path(state_def, current_input, ctx), ctx)
if state_type == "Fail":
- raise _ExecutionError(
- state_def.get("Error", "States.Fail"),
- state_def.get("Cause", ""))
+ raise _ExecutionError(*_fail_state_error(state_def, current_input, ctx))
handler_fn = {
"Pass": _execute_pass,
@@ -2534,6 +2589,32 @@ def _apply_jsonata_template(value, raw_input, ctx=None, result=None, error_outpu
return value
+def _jsonata_state_field(state_def, field, expected, raw_input, ctx):
+ """A state field that may be a JSONata expression, evaluated and held to
+ the JSON type AWS requires of it."""
+ value = _apply_jsonata_template(state_def[field], raw_input, ctx)
+ if _json_type(value) != expected:
+ raise _ExecutionError(
+ "States.QueryEvaluationError",
+ f"The JSONata expression specified for the field '{field}' returned an "
+ f"unexpected result type. Expected '{expected}', but was "
+ f"'{_json_type(value)}' for value: {json.dumps(value)}")
+ return value
+
+
+def _json_type(value):
+ """The JSON type name AWS reports for a JSONata result."""
+ if value is None:
+ return "null"
+ if isinstance(value, bool):
+ return "boolean"
+ if _is_num(value):
+ return "number"
+ if isinstance(value, str):
+ return "string"
+ return "array" if isinstance(value, list) else "object"
+
+
def _evaluate_jsonata(expression, raw_input, ctx=None, result=None, error_output=None):
"""Evaluate a Step Functions JSONata expression (the text inside `{% %}`).
diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.py
index 8b939eed..f37ca8b2 100644
--- a/tests/test_stepfunctions.py
+++ b/tests/test_stepfunctions.py
@@ -8493,6 +8493,130 @@ def test_sfn_jsonata_tomillis_still_parses_iso(sfn):
sfn.delete_state_machine(stateMachineArn=sm)
+_JSONATA_FAIL = {
+ "Type": "Fail",
+ "QueryLanguage": "JSONata",
+ "Error": "{% $states.input.error %}",
+ "Cause": "{% 'value was: ' & $states.input.v %}",
+}
+
+
+def test_sfn_jsonata_fail_evaluates_error_and_cause(sfn_sync):
+ """A JSONata Fail state evaluates its Error and Cause, in a Parallel branch
+ and through TestState too, and refuses a result that is not a string."""
+ payload = json.dumps({"error": "Example.Error", "v": "abc"})
+ evaluated = ("FAILED", "Example.Error", "value was: abc")
+
+ with _machine(sfn_sync, "Stop", {"Stop": _JSONATA_FAIL}) as arn:
+ resp = sfn_sync.start_sync_execution(stateMachineArn=arn, input=payload)
+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated
+
+ resp = sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"error": 42, "v": "abc"}))
+ assert (resp["status"], resp["error"]) == ("FAILED", "States.QueryEvaluationError")
+ assert resp["cause"] == (
+ "The JSONata expression specified for the field 'Error' returned an unexpected "
+ "result type. Expected 'string', but was 'number' for value: 42")
+
+ branch = {"StartAt": "Stop", "States": {"Stop": _JSONATA_FAIL}}
+ with _machine(sfn_sync, "Fan", {
+ "Fan": {"Type": "Parallel", "Branches": [branch], "End": True},
+ }) as arn:
+ resp = sfn_sync.start_sync_execution(stateMachineArn=arn, input=payload)
+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated
+
+ resp = sfn_sync.test_state(
+ definition=json.dumps(_JSONATA_FAIL),
+ input=payload,
+ roleArn="arn:aws:iam::000000000000:role/r",
+ )
+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated
+
+
+def test_sfn_fail_error_path_and_cause_path(sfn_sync):
+ """A JSONPath Fail state takes its error and cause from ErrorPath and
+ CausePath, a reference path or an intrinsic, which must yield a string."""
+ with _machine(sfn_sync, "Stop", {"Stop": {
+ "Type": "Fail",
+ "ErrorPath": "$.error",
+ "CausePath": "States.Format('cause: {}', $.cause)",
+ }}) as arn:
+ resp = sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"error": "Path.Error", "cause": "y"}))
+ assert (resp["status"], resp["error"], resp["cause"]) == (
+ "FAILED", "Path.Error", "cause: y")
+
+ resp = sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"error": 42, "cause": "y"}))
+ assert (resp["status"], resp["error"], resp["cause"]) == (
+ "FAILED", "States.Runtime",
+ "The value referenced by ErrorPath must be a string, but the value was: 42")
+
+
+def test_sfn_jsonata_wait_evaluates_seconds(sfn_sync):
+ """A JSONata Wait evaluates a Seconds expression, which must be a
+ non-negative integer, and applies its Output and Assign."""
+ with _machine(sfn_sync, "Pause", {
+ "Pause": {
+ "Type": "Wait",
+ "QueryLanguage": "JSONata",
+ "Seconds": "{% $states.input.s %}",
+ "Output": "{% {'out': $states.input.s} %}",
+ "Assign": {"seen": "{% $states.input.s %}"},
+ "Next": "After",
+ },
+ "After": {
+ "Type": "Pass",
+ "QueryLanguage": "JSONata",
+ "Output": "{% {'input': $states.input, 'seen': $seen} %}",
+ "End": True,
+ },
+ }) as arn:
+ def run(seconds):
+ return sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"s": seconds}))
+
+ resp = run(0)
+ assert resp["status"] == "SUCCEEDED"
+ assert json.loads(resp["output"]) == {"input": {"out": 0}, "seen": 0}
+
+ for seconds, cause in (
+ (1.5, "The Seconds field cannot be parsed as an integer: 1.5"),
+ (-1, "Seconds cannot be negative but was: -1"),
+ ("1", "The JSONata expression specified for the field 'Seconds' returned an "
+ "unexpected result type. Expected 'number', but was 'string' for value: \"1\""),
+ ):
+ resp = run(seconds)
+ assert (resp["status"], resp["error"], resp["cause"]) == (
+ "FAILED", "States.QueryEvaluationError", cause)
+
+
+def test_sfn_jsonata_wait_evaluates_timestamp(sfn_sync):
+ """A JSONata Wait evaluates a Timestamp expression, which must be a
+ parsable timestamp string."""
+ with _machine(sfn_sync, "Pause", {"Pause": {
+ "Type": "Wait",
+ "QueryLanguage": "JSONata",
+ "Timestamp": "{% $states.input.t %}",
+ "End": True,
+ }}) as arn:
+ def run(timestamp):
+ return sfn_sync.start_sync_execution(
+ stateMachineArn=arn, input=json.dumps({"t": timestamp}))
+
+ assert run("2020-01-01T00:00:00Z")["status"] == "SUCCEEDED"
+
+ for timestamp, cause in (
+ ("not a time", "The Timestamp field cannot be parsed as an ISO-8601 extended "
+ "offset date-time format string: not a time"),
+ (5, "The JSONata expression specified for the field 'Timestamp' returned an "
+ "unexpected result type. Expected 'string', but was 'number' for value: 5"),
+ ):
+ resp = run(timestamp)
+ assert (resp["status"], resp["error"], resp["cause"]) == (
+ "FAILED", "States.QueryEvaluationError", cause)
+
+
def test_sfn_choice_numeric_path_dynamic_batch_loop(sfn):
"""A dynamic batch loop guarded by NumericGreaterThanEqualsPath must exit once
the cursor reaches the batch size. Before the *Path relational operators were