Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(durable lambda): add extra 4 missing ops (#1643)
Nahuel Nucerab1a12593e939
diff --git a/ministack/services/lambda_durable.py b/ministack/services/lambda_durable.pyindex ee92f614..c8ba786c 100644--- a/ministack/services/lambda_durable.py+++ b/ministack/services/lambda_durable.py@@ -361,7 +361,8 @@ def _emit_history_event(rec: dict, event_type: str, details_key: str, details: ddef create_execution_for_invoke(function_arn: str, version: str,input_payload: str,name: str | None = None,- trace_id: str | None = None) -> dict:+ trace_id: str | None = None,+ execution_timeout: int | None = None) -> dict:"""Spin up a new durable execution and return its record. The Lambdaruntime is expected to read the ARN from the AWS_DURABLE_EXECUTION_ARNenv var and call Checkpoint/GetState through the regular Lambda endpoint."""@@ -397,10 +398,19 @@ def create_execution_for_invoke(function_arn: str, version: str,"History": [],"NextEventId": 0,}+ if execution_timeout:+ # DurableConfig.ExecutionTimeout caps the whole execution (the AWS+ # field is seconds, 1..31622400). Stored as an absolute epoch deadline+ # so it survives restarts; enforced by the resume scheduler. No+ # default is invented — an unset field leaves the execution uncapped,+ # exactly as broadly as before.+ rec["_TimeoutDeadline"] = rec["StartTimestamp"] + int(execution_timeout)_executions[arn] = rec_emit_history_event(rec, "ExecutionStarted", "ExecutionStartedDetails", {"Input": {"Payload": input_payload or "", "Truncated": False},})+ if execution_timeout:+ schedule_resume(arn)return rec@@ -409,6 +419,10 @@ def mark_execution_completed(arn: str, result_payload: str | None,rec = _executions.get(arn)if not rec:return+ if rec.get("Status") != "RUNNING":+ # Stopped or timed out while the invocation was in flight: the late+ # result must not overwrite the terminal state or append to history.+ returnrec["EndTimestamp"] = _now()if error:rec["Status"] = "FAILED"@@ -424,6 +438,55 @@ def mark_execution_completed(arn: str, result_payload: str | None,})+def record_invocation_completed(arn: str, start_ts: float, request_id: str,+ error: dict | None = None) -> None:+ """One InvocationCompleted history event per handler invocation (initial+ and every replay), the shape the model defines: StartTimestamp,+ EndTimestamp, RequestId, and Error when the invocation itself failed."""+ rec = _executions.get(arn)+ if not rec:+ return+ details = {+ "StartTimestamp": start_ts,+ "EndTimestamp": _now(),+ "RequestId": request_id,+ }+ if error:+ details["Error"] = {"Payload": error, "Truncated": False}+ _emit_history_event(rec, "InvocationCompleted", "InvocationCompletedDetails",+ details)+++def _time_out_execution(rec: dict) -> None:+ """Enforce DurableConfig.ExecutionTimeout: the execution lands TIMED_OUT+ with an ExecutionTimedOut event, and every in-flight chained invoke is+ marked TIMED_OUT with its own ChainedInvokeTimedOut event."""+ if rec.get("Status") != "RUNNING":+ return+ err = {+ "ErrorType": "DurableExecutionTimedOut",+ "ErrorMessage": "The durable execution exceeded its configured "+ "ExecutionTimeout.",+ "ErrorData": "",+ "StackTrace": [],+ }+ for op in rec.get("Operations", []):+ if op.get("Type") == "CHAINED_INVOKE" and op.get("Status") == "STARTED":+ op["Status"] = "TIMED_OUT"+ op["EndTimestamp"] = _now()+ op.setdefault("ChainedInvokeDetails", {})["Error"] = err+ _emit_history_event(rec, "ChainedInvokeTimedOut",+ "ChainedInvokeTimedOutDetails",+ {"Error": {"Payload": err, "Truncated": False}},+ event_id=op.get("Id"))+ rec["Status"] = "TIMED_OUT"+ rec["EndTimestamp"] = _now()+ rec["Error"] = err+ _emit_history_event(rec, "ExecutionTimedOut", "ExecutionTimedOutDetails", {+ "Error": {"Payload": err, "Truncated": False},+ })++# ---------------------------------------------------------------------------# Handlers — wired in from lambda_svc.handle_request based on path matching.# ---------------------------------------------------------------------------@@ -492,6 +555,10 @@ def _append_chained_result(parent_rec: dict, op_id: str, success: bool,result: str | None, err: dict | None) -> None:"""Apply the child invocation's outcome onto the parent ChainedInvokeoperation in the operation log + emit a history event."""+ if parent_rec.get("Status") != "RUNNING":+ # The execution was stopped or timed out while the child ran; its op+ # already carries the terminal status and event.+ returnfor op in parent_rec["Operations"]:if op.get("Id") == op_id and op.get("Type") == "CHAINED_INVOKE":op["Status"] = "SUCCEEDED" if success else "FAILED"@@ -672,11 +739,11 @@ def _apply_update(rec: dict, upd: dict) -> None:details["Result"] = payloadif err is not None:details["Error"] = err- # On START, kick off the child function invocation asynchronously so- # downstream durable workflows actually run (item #3 in the parity gap).+ # The child fires below, after the history mirror, so the+ # ChainedInvokeStarted event is on the log before a fast child can+ # append its ChainedInvokeSucceeded (history order matters to+ # replaying SDKs).ci_opts = upd.get("ChainedInvokeOptions") or {}- if action == "START" and ci_opts.get("FunctionName"):- _fire_chained_invoke(rec, op_id, ci_opts, payload)elif op_type == "EXECUTION":details = existing.setdefault("ExecutionDetails", {})if rec.get("InputPayload"):@@ -701,6 +768,14 @@ def _apply_update(rec: dict, upd: dict) -> None:{"Result": {"Payload": payload or "", "Truncated": False}}),("CALLBACK", "FAIL"): ("CallbackFailed", "CallbackFailedDetails",{"Error": {"Payload": err or {}, "Truncated": False}}),+ # ChainedInvokeStarted is a real history EventType on the model;+ # without it a client polling GetDurableExecutionHistory cannot see+ # an in-flight child at all.+ ("CHAINED_INVOKE", "START"): (+ "ChainedInvokeStarted", "ChainedInvokeStartedDetails",+ {"FunctionName": upd.get("ChainedInvokeOptions", {}).get("FunctionName", ""),+ "Input": {"Payload": payload or "", "Truncated": False},+ "DurableExecutionArn": rec.get("DurableExecutionArn", "")}),("CONTEXT", "START"): ("ContextStarted", "ContextStartedDetails", {}),("CONTEXT", "SUCCEED"): ("ContextSucceeded", "ContextSucceededDetails",{"Result": {"Payload": payload or "", "Truncated": False}}),@@ -714,6 +789,11 @@ def _apply_update(rec: dict, upd: dict) -> None:name=name, parent_id=parent_id, sub_type=sub_type,event_id=op_id)+ # Kick off the chained child only after its Started event is on the log.+ if op_type == "CHAINED_INVOKE" and action == "START" and (+ upd.get("ChainedInvokeOptions") or {}).get("FunctionName"):+ _fire_chained_invoke(rec, op_id, upd.get("ChainedInvokeOptions") or {}, payload)+def handle_get_state(arn_path: str, query_params: dict) -> tuple:rec, err = _require_execution(arn_path)@@ -905,6 +985,20 @@ def handle_stop(arn_path: str, body: bytes) -> tuple:"ErrorData": data.get("ErrorData") or "","StackTrace": data.get("StackTrace") or [],}+ # A stop reaches the in-flight chained invokes too: each STARTED+ # CHAINED_INVOKE op lands STOPPED with its own history event+ # (ChainedInvokeStopped is a real EventType on the model), and the late+ # child result is discarded by the RUNNING guard in+ # _append_chained_result instead of appending onto a stopped execution.+ for op in rec.get("Operations", []):+ if op.get("Type") == "CHAINED_INVOKE" and op.get("Status") == "STARTED":+ op["Status"] = "STOPPED"+ op["EndTimestamp"] = _now()+ op.setdefault("ChainedInvokeDetails", {})["Error"] = rec["Error"]+ _emit_history_event(rec, "ChainedInvokeStopped",+ "ChainedInvokeStoppedDetails",+ {"Error": {"Payload": rec["Error"], "Truncated": False}},+ event_id=op.get("Id"))_emit_history_event(rec, "ExecutionStopped", "ExecutionStoppedDetails", {"Error": {"Payload": rec["Error"], "Truncated": False},})@@ -1169,6 +1263,12 @@ def schedule_resume(arn: str, account_id: str | None = None,if not rec or rec.get("Status") != "RUNNING":return Falseexpiry = _next_expiry(rec)+ deadline = rec.get("_TimeoutDeadline")+ if deadline is not None:+ # The execution-timeout deadline is a wake-up like any other: the+ # earliest pending timer wins, and an execution with no waits at all+ # still gets its timeout enforced.+ expiry = deadline if expiry is None else min(expiry, deadline)# When called from a Send*Callback handler we want to wake the execution# immediately so the SDK observes the resolution on its next replay.has_resolved_callback = any(@@ -1252,6 +1352,13 @@ def _resume_execution(arn: str, account_id: str = "000000000000",if not rec or rec.get("Status") != "RUNNING":returnnow = _now()+ deadline = rec.get("_TimeoutDeadline")+ if deadline is not None and now >= deadline:+ # DurableConfig.ExecutionTimeout elapsed: the execution lands+ # TIMED_OUT instead of being re-invoked, and every in-flight chained+ # invoke is timed out with it.+ _time_out_execution(rec)+ returnanything_elapsed = Falsehas_resolved_callback = any(op.get("Type") == "CALLBACK" and op.get("Status") in ("SUCCEEDED", "FAILED")diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.pyindex be78e8b5..9cbf85ca 100644--- a/ministack/services/lambda_svc.py+++ b/ministack/services/lambda_svc.py@@ -1261,7 +1261,14 @@ def invoke_durable_resume(function_name: str, durable_arn: str, original_event:},}try:+ _replay_started = time.time()result = _execute_function(func, resume_event)+ lambda_durable.record_invocation_completed(+ durable_arn,+ start_ts=_replay_started,+ request_id=new_uuid(),+ error=(result.get("body") if result.get("error") else None),+ )# If the resume still returns PENDING, schedule the next wakeup.try:payload = result.get("body")@@ -2840,6 +2847,8 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | Nfunction_arn=_func_arn(name),version=executed_version,input_payload=event_payload,+ execution_timeout=(func.get("config", {}) or {}).get(+ "DurableConfig", {}).get("ExecutionTimeout"),)durable_arn = rec["DurableExecutionArn"]_durable_ctx.set({@@ -2899,6 +2908,15 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | N_de_rec = lambda_durable._executions.get(durable_arn)if _de_rec:resp_headers["X-Amz-Durable-Checkpoint-Token"] = _de_rec["CheckpointToken"]+ # Every handler invocation of a durable execution ends with an+ # InvocationCompleted history event (initial and each replay), before+ # any terminal ExecutionSucceeded/Failed the payload may cause.+ lambda_durable.record_invocation_completed(+ durable_arn,+ start_ts=(_de_rec or {}).get("StartTimestamp") or time.time(),+ request_id=new_uuid(),+ error=(result.get("body") if result.get("error") else None),+ )# Inspect the SDK's return value: PENDING → schedule the next wakeup# from the latest WAIT timestamp; SUCCEEDED/FAILED → mark terminal.try:diff --git a/tests/test_lambda.py b/tests/test_lambda.pyindex 68a9c7a9..3d4851db 100644--- a/tests/test_lambda.py+++ b/tests/test_lambda.py@@ -8011,6 +8011,112 @@ def test_lambda_durable_stop(lam):lam.delete_function(FunctionName=fname)+def test_lambda_durable_invocation_completed_recorded(lam):+ """Every handler invocation of a durable execution ends with an+ InvocationCompleted history event (a real EventType on the model,+ previously never emitted), carrying a RequestId and both timestamps."""+ fname, _, rec = _create_durable_execution_directly(lam)+ try:+ from urllib.parse import quote+ arn = quote(rec["DurableExecutionArn"], safe="/:$")+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")+ assert code == 200+ completed = [e for e in body["Events"] if e["EventType"] == "InvocationCompleted"]+ assert completed, [e["EventType"] for e in body["Events"]]+ details = completed[0]["InvocationCompletedDetails"]+ assert details["RequestId"]+ assert details["EndTimestamp"] >= details["StartTimestamp"] - 1+ finally:+ lam.delete_function(FunctionName=fname)+++def test_lambda_durable_execution_timeout_enforced(lam):+ """DurableConfig.ExecutionTimeout (a real field, seconds) caps the whole+ execution: it was stored and ignored, so a RUNNING execution could out-+ live its cap forever. It now lands TIMED_OUT with an ExecutionTimedOut+ event once the deadline passes. The wait below is the 1-second timeout+ itself firing, not a contention budget."""+ import base64 as _b64+ fname = f"durable-timeout-{_uuid_mod.uuid4().hex[:8]}"+ zip_b64 = _b64.b64encode(_make_zip("def handler(e,c): return {}")).decode()+ _raw_durable("POST", "/2015-03-31/functions", body={+ "FunctionName": fname, "Runtime": "python3.12", "Role": _LAMBDA_ROLE,+ "Handler": "index.handler", "Code": {"ZipFile": zip_b64},+ "DurableConfig": {"Enabled": True, "ExecutionTimeout": 1},+ })+ try:+ invoke_req = urllib.request.Request(+ f"{_ms_endpoint()}/2015-03-31/functions/{fname}/invocations",+ method="POST", data=b"{}", headers={"Content-Type": "application/json"},+ )+ with urllib.request.urlopen(invoke_req) as r:+ arn_raw = r.headers.get("X-Amz-Durable-Execution-Arn")+ r.read()+ from urllib.parse import quote+ arn = quote(arn_raw, safe="/:$")+ deadline = time.time() + 10+ status = None+ while time.time() < deadline:+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}")+ status = body["Status"]+ if status != "RUNNING":+ break+ time.sleep(0.2)+ assert status == "TIMED_OUT", status+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")+ assert any(e["EventType"] == "ExecutionTimedOut" for e in hist["Events"]), (+ [e["EventType"] for e in hist["Events"]])+ finally:+ lam.delete_function(FunctionName=fname)+++def test_lambda_durable_stop_reaches_inflight_chained_invoke(lam):+ """StopDurableExecution reaches an in-flight chained invoke: the op lands+ STOPPED with a ChainedInvokeStopped event (a real EventType, previously+ never emitted), and the child's late result is discarded instead of+ appending ChainedInvokeSucceeded onto a stopped execution."""+ import base64 as _b64+ child = f"durable-stopchild-{_uuid_mod.uuid4().hex[:8]}"+ _raw_durable("POST", "/2015-03-31/functions", body={+ "FunctionName": child, "Runtime": "python3.12", "Role": _LAMBDA_ROLE,+ "Handler": "index.handler",+ "Code": {"ZipFile": _b64.b64encode(_make_zip(+ "import time\ndef handler(e,c):\n time.sleep(2)\n return {'late': True}")).decode()},+ "Timeout": 30,+ })+ fname, _, rec = _create_durable_execution_directly(lam)+ try:+ from urllib.parse import quote+ arn = quote(rec["DurableExecutionArn"], safe="/:$")+ code, _b = _raw_durable("POST", f"/2025-12-01/durable-executions/{arn}/checkpoint", body={+ "CheckpointToken": rec["CheckpointToken"],+ "Updates": [{"Id": "chain-stop", "Type": "CHAINED_INVOKE",+ "Action": "START", "Name": "slow-child",+ "ChainedInvokeOptions": {"FunctionName": child}}],+ })+ assert code == 200+ code, _b = _raw_durable("POST", f"/2025-12-01/durable-executions/{arn}/stop", body={+ "ErrorMessage": "stopping with child in flight",+ })+ assert code == 200+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")+ types = [e["EventType"] for e in hist["Events"]]+ assert "ChainedInvokeStarted" in types, types+ assert "ChainedInvokeStopped" in types, types+ # Past the child's 2s sleep: its late result must not have been applied.+ time.sleep(3)+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")+ types = [e["EventType"] for e in hist["Events"]]+ assert "ChainedInvokeSucceeded" not in types, types+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}")+ assert body["Status"] == "STOPPED"+ finally:+ lam.delete_function(FunctionName=fname)+ lam.delete_function(FunctionName=child)++def test_lambda_durable_list_by_function(lam):fname, fn_arn, rec = _create_durable_execution_directly(lam)try:@@ -8078,6 +8184,7 @@ def handler(event, context):passdef test_lambda_durable_chained_invoke_runs_child(lam):"""A CHAINED_INVOKE checkpoint update with Action=START actually spawnsthe child function and records the result back into the parent's@@ -8133,13 +8240,24 @@ def test_lambda_durable_chained_invoke_runs_child(lam):}],})assert code == 200- # The child runs in a daemon thread; give it a moment.+ # ChainedInvokeStarted is recorded synchronously when the checkpoint+ # accepts the START update, so it must already be on the log — no+ # polling, no budget.+ code, history = _raw_durable("GET",+ f"/2025-12-01/durable-executions/{arn_enc}/history")+ started = [e for e in history["Events"] if e["EventType"] == "ChainedInvokeStarted"]+ assert started, [e["EventType"] for e in history["Events"]]+ assert started[0]["ChainedInvokeStartedDetails"]["FunctionName"] == child+ # The child completes in a background thread; serial keeps the box+ # quiet so the short poll is honest. Breaking on Failed keeps a real+ # child error visible instead of reading as a timeout.import time as _timefor _ in range(30):_time.sleep(0.1)code, history = _raw_durable("GET",f"/2025-12-01/durable-executions/{arn_enc}/history")- if any(e["EventType"] == "ChainedInvokeSucceeded" for e in history.get("Events", [])):+ if any(e["EventType"] in ("ChainedInvokeSucceeded", "ChainedInvokeFailed")+ for e in history.get("Events", [])):breakevents = history["Events"]assert any(e["EventType"] == "ChainedInvokeSucceeded" for e in events), \