Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(sfn): evaluate JSONata Fail Error/Cause and Wait Seconds/Timestamp (#2167) (#2203)
Lucas9e3909dfd7e1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fec54677..60874fea 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,6 +14,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).- **Auth — header signatures are verified for IAM user and STS session keys** — under `AUTH=true` a request signed with such a key and the wrong secret, or changed after signing, was served; it now answers `SignatureDoesNotMatch` (or the service's `InvalidSignatureException`), an `Authorization` header without its `Signature` or `SignedHeaders` answers `IncompleteSignature` (Lambda `MissingAuthenticationTokenException`, S3 `AuthorizationHeaderMalformed`) and a credential date other than the request date `SignatureDoesNotMatch` (S3 `AuthorizationHeaderMalformed`), S3 unsigned, streaming and signed-chunk payloads are checked the same way, and the root key still accepts any secret. Contributed by @iot-rocket.### Fixed+- **Step Functions — JSONata Fail and Wait fields are evaluated** — a JSONata Fail state raised its `Error` and `Cause` expressions as literal text, a JSONata Wait failed on a `Seconds` expression, ignored a `Timestamp` expression and dropped its `Output` and `Assign`, and JSONPath `ErrorPath`/`CausePath` were ignored; they are now evaluated, with AWS's `States.QueryEvaluationError` or `States.Runtime` for a result of the wrong type, a `Seconds` that is not a non-negative integer and an unparsable `Timestamp`.- **S3 — presigned URLs keep AWS's time limits under `AUTH=true`** — an `X-Amz-Expires` over 604800 seconds or not a number, or a malformed `X-Amz-Date`, answers 400 `AuthorizationQueryParametersError`, and a URL dated more than 15 minutes ahead answers 403 `AccessDenied` `Request is not yet valid`, where they were served. Contributed by @iot-rocket.- **Auth — credential failures in the service's protocol** — under `AUTH=true` an unknown, inactive or expired access key answered `UnrecognizedClientException` 403 from every service; it now gets the code, status and `Content-Type` AWS sends for the service's protocol, for example `InvalidClientTokenId` 403 from IAM and SQS, `UnrecognizedClientException` 400 from DynamoDB, `InvalidAccessKeyId` 403 from S3 and `AuthFailure` 401 from EC2. Contributed by @iot-rocket.### Fixeddiff --git a/ministack/services/stepfunctions.py b/ministack/services/stepfunctions.pyindex dfcd9f23..f9fb1ea1 100644--- a/ministack/services/stepfunctions.py+++ b/ministack/services/stepfunctions.py@@ -1276,11 +1276,8 @@ def _test_state(data):result = {"status": "SUCCEEDED", "output": json.dumps(output)}elif state_type == "Fail":- result = {- "status": "FAILED",- "error": state_def.get("Error", "States.Fail"),- "cause": state_def.get("Cause", ""),- }+ error, cause = _fail_state_error(state_def, input_data, ctx)+ result = {"status": "FAILED", "error": error, "cause": cause}elif state_type == "Task":effective = _apply_input_path(state_def, input_data, ctx)@@ -1576,9 +1573,7 @@ def _run_execution(exec_arn):continueif state_type == "Fail":- raise _ExecutionError(- state_def.get("Error", "States.Fail"),- state_def.get("Cause", ""))+ raise _ExecutionError(*_fail_state_error(state_def, current_input, ctx))handler_fn = {"Pass": _execute_pass,@@ -2181,6 +2176,17 @@ def _evaluate_rule(rule, data, ctx=None):# ---------------------------------------------------------------------------def _execute_wait(state_def, raw_input, ctx=None):+ if _state_query_language(state_def, ctx) == "JSONata":+ # JSONata Wait: Seconds and Timestamp may be expressions, and Output+ # and Assign replace the JSONPath chain, as for Pass.+ if "Seconds" in state_def:+ _scaled_sleep(_jsonata_wait_seconds(state_def, raw_input, ctx))+ elif "Timestamp" in state_def:+ _sleep_until(_jsonata_wait_timestamp(state_def, raw_input, ctx))+ output = _apply_jsonata_output(state_def, raw_input, ctx, default=raw_input)+ _apply_state_assign(state_def, raw_input, ctx)+ return output, _next_or_end(state_def)+effective = _apply_input_path(state_def, raw_input, ctx)if "Seconds" in state_def:@@ -2200,6 +2206,29 @@ def _execute_wait(state_def, raw_input, ctx=None):return output, _next_or_end(state_def)+def _jsonata_wait_seconds(state_def, raw_input, ctx):+ seconds = _jsonata_state_field(state_def, "Seconds", "number", raw_input, ctx)+ if not float(seconds).is_integer():+ raise _ExecutionError(+ "States.QueryEvaluationError",+ f"The Seconds field cannot be parsed as an integer: {json.dumps(seconds)}")+ if seconds < 0:+ raise _ExecutionError(+ "States.QueryEvaluationError",+ f"Seconds cannot be negative but was: {json.dumps(seconds)}")+ return seconds+++def _jsonata_wait_timestamp(state_def, raw_input, ctx):+ timestamp = _jsonata_state_field(state_def, "Timestamp", "string", raw_input, ctx)+ if _parse_ts(timestamp) is None:+ raise _ExecutionError(+ "States.QueryEvaluationError",+ "The Timestamp field cannot be parsed as an ISO-8601 extended offset "+ f"date-time format string: {timestamp}")+ return timestamp++def _scaled_sleep(seconds):scaled = seconds * _SFN_WAIT_SCALEif scaled > 0:@@ -2216,6 +2245,34 @@ def _sleep_until(iso_ts):pass+# ---------------------------------------------------------------------------+# Fail state+# ---------------------------------------------------------------------------++def _fail_state_error(state_def, raw_input, ctx=None):+ """The error and cause a Fail state ends with. JSONata states may give+ either as an expression; JSONPath states may select either from the+ input with ErrorPath or CausePath. Both must be strings."""+ return (_fail_state_field(state_def, "Error", "States.Fail", raw_input, ctx),+ _fail_state_field(state_def, "Cause", "", raw_input, ctx))+++def _fail_state_field(state_def, field, default, raw_input, ctx):+ if field in state_def and _state_query_language(state_def, ctx) == "JSONata":+ return _jsonata_state_field(state_def, field, "string", raw_input, ctx)+ path = state_def.get(f"{field}Path")+ if path is None:+ return state_def.get(field, default)+ value = (_evaluate_intrinsic(path, raw_input, ctx) if path.startswith("States.")+ else _resolve_path_or_ctx(path, raw_input, ctx))+ if not isinstance(value, str):+ raise _ExecutionError(+ "States.Runtime",+ f"The value referenced by {field}Path must be a string, "+ f"but the value was: {json.dumps(value)}")+ return value++# ---------------------------------------------------------------------------# Parallel state# ---------------------------------------------------------------------------@@ -2385,9 +2442,7 @@ def _run_sub_machine(states, start_at, input_data, execution, ctx):return _apply_output_path(state_def,_apply_input_path(state_def, current_input, ctx), ctx)if state_type == "Fail":- raise _ExecutionError(- state_def.get("Error", "States.Fail"),- state_def.get("Cause", ""))+ raise _ExecutionError(*_fail_state_error(state_def, current_input, ctx))handler_fn = {"Pass": _execute_pass,@@ -2534,6 +2589,32 @@ def _apply_jsonata_template(value, raw_input, ctx=None, result=None, error_outpureturn value+def _jsonata_state_field(state_def, field, expected, raw_input, ctx):+ """A state field that may be a JSONata expression, evaluated and held to+ the JSON type AWS requires of it."""+ value = _apply_jsonata_template(state_def[field], raw_input, ctx)+ if _json_type(value) != expected:+ raise _ExecutionError(+ "States.QueryEvaluationError",+ f"The JSONata expression specified for the field '{field}' returned an "+ f"unexpected result type. Expected '{expected}', but was "+ f"'{_json_type(value)}' for value: {json.dumps(value)}")+ return value+++def _json_type(value):+ """The JSON type name AWS reports for a JSONata result."""+ if value is None:+ return "null"+ if isinstance(value, bool):+ return "boolean"+ if _is_num(value):+ return "number"+ if isinstance(value, str):+ return "string"+ return "array" if isinstance(value, list) else "object"++def _evaluate_jsonata(expression, raw_input, ctx=None, result=None, error_output=None):"""Evaluate a Step Functions JSONata expression (the text inside `{% %}`).diff --git a/tests/test_stepfunctions.py b/tests/test_stepfunctions.pyindex 8b939eed..f37ca8b2 100644--- a/tests/test_stepfunctions.py+++ b/tests/test_stepfunctions.py@@ -8493,6 +8493,130 @@ def test_sfn_jsonata_tomillis_still_parses_iso(sfn):sfn.delete_state_machine(stateMachineArn=sm)+_JSONATA_FAIL = {+ "Type": "Fail",+ "QueryLanguage": "JSONata",+ "Error": "{% $states.input.error %}",+ "Cause": "{% 'value was: ' & $states.input.v %}",+}+++def test_sfn_jsonata_fail_evaluates_error_and_cause(sfn_sync):+ """A JSONata Fail state evaluates its Error and Cause, in a Parallel branch+ and through TestState too, and refuses a result that is not a string."""+ payload = json.dumps({"error": "Example.Error", "v": "abc"})+ evaluated = ("FAILED", "Example.Error", "value was: abc")++ with _machine(sfn_sync, "Stop", {"Stop": _JSONATA_FAIL}) as arn:+ resp = sfn_sync.start_sync_execution(stateMachineArn=arn, input=payload)+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated++ resp = sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"error": 42, "v": "abc"}))+ assert (resp["status"], resp["error"]) == ("FAILED", "States.QueryEvaluationError")+ assert resp["cause"] == (+ "The JSONata expression specified for the field 'Error' returned an unexpected "+ "result type. Expected 'string', but was 'number' for value: 42")++ branch = {"StartAt": "Stop", "States": {"Stop": _JSONATA_FAIL}}+ with _machine(sfn_sync, "Fan", {+ "Fan": {"Type": "Parallel", "Branches": [branch], "End": True},+ }) as arn:+ resp = sfn_sync.start_sync_execution(stateMachineArn=arn, input=payload)+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated++ resp = sfn_sync.test_state(+ definition=json.dumps(_JSONATA_FAIL),+ input=payload,+ roleArn="arn:aws:iam::000000000000:role/r",+ )+ assert (resp["status"], resp["error"], resp["cause"]) == evaluated+++def test_sfn_fail_error_path_and_cause_path(sfn_sync):+ """A JSONPath Fail state takes its error and cause from ErrorPath and+ CausePath, a reference path or an intrinsic, which must yield a string."""+ with _machine(sfn_sync, "Stop", {"Stop": {+ "Type": "Fail",+ "ErrorPath": "$.error",+ "CausePath": "States.Format('cause: {}', $.cause)",+ }}) as arn:+ resp = sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"error": "Path.Error", "cause": "y"}))+ assert (resp["status"], resp["error"], resp["cause"]) == (+ "FAILED", "Path.Error", "cause: y")++ resp = sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"error": 42, "cause": "y"}))+ assert (resp["status"], resp["error"], resp["cause"]) == (+ "FAILED", "States.Runtime",+ "The value referenced by ErrorPath must be a string, but the value was: 42")+++def test_sfn_jsonata_wait_evaluates_seconds(sfn_sync):+ """A JSONata Wait evaluates a Seconds expression, which must be a+ non-negative integer, and applies its Output and Assign."""+ with _machine(sfn_sync, "Pause", {+ "Pause": {+ "Type": "Wait",+ "QueryLanguage": "JSONata",+ "Seconds": "{% $states.input.s %}",+ "Output": "{% {'out': $states.input.s} %}",+ "Assign": {"seen": "{% $states.input.s %}"},+ "Next": "After",+ },+ "After": {+ "Type": "Pass",+ "QueryLanguage": "JSONata",+ "Output": "{% {'input': $states.input, 'seen': $seen} %}",+ "End": True,+ },+ }) as arn:+ def run(seconds):+ return sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"s": seconds}))++ resp = run(0)+ assert resp["status"] == "SUCCEEDED"+ assert json.loads(resp["output"]) == {"input": {"out": 0}, "seen": 0}++ for seconds, cause in (+ (1.5, "The Seconds field cannot be parsed as an integer: 1.5"),+ (-1, "Seconds cannot be negative but was: -1"),+ ("1", "The JSONata expression specified for the field 'Seconds' returned an "+ "unexpected result type. Expected 'number', but was 'string' for value: \"1\""),+ ):+ resp = run(seconds)+ assert (resp["status"], resp["error"], resp["cause"]) == (+ "FAILED", "States.QueryEvaluationError", cause)+++def test_sfn_jsonata_wait_evaluates_timestamp(sfn_sync):+ """A JSONata Wait evaluates a Timestamp expression, which must be a+ parsable timestamp string."""+ with _machine(sfn_sync, "Pause", {"Pause": {+ "Type": "Wait",+ "QueryLanguage": "JSONata",+ "Timestamp": "{% $states.input.t %}",+ "End": True,+ }}) as arn:+ def run(timestamp):+ return sfn_sync.start_sync_execution(+ stateMachineArn=arn, input=json.dumps({"t": timestamp}))++ assert run("2020-01-01T00:00:00Z")["status"] == "SUCCEEDED"++ for timestamp, cause in (+ ("not a time", "The Timestamp field cannot be parsed as an ISO-8601 extended "+ "offset date-time format string: not a time"),+ (5, "The JSONata expression specified for the field 'Timestamp' returned an "+ "unexpected result type. Expected 'string', but was 'number' for value: 5"),+ ):+ resp = run(timestamp)+ assert (resp["status"], resp["error"], resp["cause"]) == (+ "FAILED", "States.QueryEvaluationError", cause)++def test_sfn_choice_numeric_path_dynamic_batch_loop(sfn):"""A dynamic batch loop guarded by NumericGreaterThanEqualsPath must exit oncethe cursor reaches the batch size. Before the *Path relational operators were