Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(durable lambda): add extra 4 missing ops (#1643)

Nahuel Nucerab1a12593e939
diff --git a/ministack/services/lambda_durable.py b/ministack/services/lambda_durable.py
index ee92f614..c8ba786c 100644
--- a/ministack/services/lambda_durable.py
+++ b/ministack/services/lambda_durable.py
@@ -361,7 +361,8 @@ def _emit_history_event(rec: dict, event_type: str, details_key: str, details: d
def create_execution_for_invoke(function_arn: str, version: str,
input_payload: str,
name: str | None = None,
- trace_id: str | None = None) -> dict:
+ trace_id: str | None = None,
+ execution_timeout: int | None = None) -> dict:
"""Spin up a new durable execution and return its record. The Lambda
runtime is expected to read the ARN from the AWS_DURABLE_EXECUTION_ARN
env var and call Checkpoint/GetState through the regular Lambda endpoint."""
@@ -397,10 +398,19 @@ def create_execution_for_invoke(function_arn: str, version: str,
"History": [],
"NextEventId": 0,
}
+ if execution_timeout:
+ # DurableConfig.ExecutionTimeout caps the whole execution (the AWS
+ # field is seconds, 1..31622400). Stored as an absolute epoch deadline
+ # so it survives restarts; enforced by the resume scheduler. No
+ # default is invented — an unset field leaves the execution uncapped,
+ # exactly as broadly as before.
+ rec["_TimeoutDeadline"] = rec["StartTimestamp"] + int(execution_timeout)
_executions[arn] = rec
_emit_history_event(rec, "ExecutionStarted", "ExecutionStartedDetails", {
"Input": {"Payload": input_payload or "", "Truncated": False},
})
+ if execution_timeout:
+ schedule_resume(arn)
return rec
@@ -409,6 +419,10 @@ def mark_execution_completed(arn: str, result_payload: str | None,
rec = _executions.get(arn)
if not rec:
return
+ if rec.get("Status") != "RUNNING":
+ # Stopped or timed out while the invocation was in flight: the late
+ # result must not overwrite the terminal state or append to history.
+ return
rec["EndTimestamp"] = _now()
if error:
rec["Status"] = "FAILED"
@@ -424,6 +438,55 @@ def mark_execution_completed(arn: str, result_payload: str | None,
})
+def record_invocation_completed(arn: str, start_ts: float, request_id: str,
+ error: dict | None = None) -> None:
+ """One InvocationCompleted history event per handler invocation (initial
+ and every replay), the shape the model defines: StartTimestamp,
+ EndTimestamp, RequestId, and Error when the invocation itself failed."""
+ rec = _executions.get(arn)
+ if not rec:
+ return
+ details = {
+ "StartTimestamp": start_ts,
+ "EndTimestamp": _now(),
+ "RequestId": request_id,
+ }
+ if error:
+ details["Error"] = {"Payload": error, "Truncated": False}
+ _emit_history_event(rec, "InvocationCompleted", "InvocationCompletedDetails",
+ details)
+
+
+def _time_out_execution(rec: dict) -> None:
+ """Enforce DurableConfig.ExecutionTimeout: the execution lands TIMED_OUT
+ with an ExecutionTimedOut event, and every in-flight chained invoke is
+ marked TIMED_OUT with its own ChainedInvokeTimedOut event."""
+ if rec.get("Status") != "RUNNING":
+ return
+ err = {
+ "ErrorType": "DurableExecutionTimedOut",
+ "ErrorMessage": "The durable execution exceeded its configured "
+ "ExecutionTimeout.",
+ "ErrorData": "",
+ "StackTrace": [],
+ }
+ for op in rec.get("Operations", []):
+ if op.get("Type") == "CHAINED_INVOKE" and op.get("Status") == "STARTED":
+ op["Status"] = "TIMED_OUT"
+ op["EndTimestamp"] = _now()
+ op.setdefault("ChainedInvokeDetails", {})["Error"] = err
+ _emit_history_event(rec, "ChainedInvokeTimedOut",
+ "ChainedInvokeTimedOutDetails",
+ {"Error": {"Payload": err, "Truncated": False}},
+ event_id=op.get("Id"))
+ rec["Status"] = "TIMED_OUT"
+ rec["EndTimestamp"] = _now()
+ rec["Error"] = err
+ _emit_history_event(rec, "ExecutionTimedOut", "ExecutionTimedOutDetails", {
+ "Error": {"Payload": err, "Truncated": False},
+ })
+
+
# ---------------------------------------------------------------------------
# Handlers — wired in from lambda_svc.handle_request based on path matching.
# ---------------------------------------------------------------------------
@@ -492,6 +555,10 @@ def _append_chained_result(parent_rec: dict, op_id: str, success: bool,
result: str | None, err: dict | None) -> None:
"""Apply the child invocation's outcome onto the parent ChainedInvoke
operation in the operation log + emit a history event."""
+ if parent_rec.get("Status") != "RUNNING":
+ # The execution was stopped or timed out while the child ran; its op
+ # already carries the terminal status and event.
+ return
for op in parent_rec["Operations"]:
if op.get("Id") == op_id and op.get("Type") == "CHAINED_INVOKE":
op["Status"] = "SUCCEEDED" if success else "FAILED"
@@ -672,11 +739,11 @@ def _apply_update(rec: dict, upd: dict) -> None:
details["Result"] = payload
if err is not None:
details["Error"] = err
- # On START, kick off the child function invocation asynchronously so
- # downstream durable workflows actually run (item #3 in the parity gap).
+ # The child fires below, after the history mirror, so the
+ # ChainedInvokeStarted event is on the log before a fast child can
+ # append its ChainedInvokeSucceeded (history order matters to
+ # replaying SDKs).
ci_opts = upd.get("ChainedInvokeOptions") or {}
- if action == "START" and ci_opts.get("FunctionName"):
- _fire_chained_invoke(rec, op_id, ci_opts, payload)
elif op_type == "EXECUTION":
details = existing.setdefault("ExecutionDetails", {})
if rec.get("InputPayload"):
@@ -701,6 +768,14 @@ def _apply_update(rec: dict, upd: dict) -> None:
{"Result": {"Payload": payload or "", "Truncated": False}}),
("CALLBACK", "FAIL"): ("CallbackFailed", "CallbackFailedDetails",
{"Error": {"Payload": err or {}, "Truncated": False}}),
+ # ChainedInvokeStarted is a real history EventType on the model;
+ # without it a client polling GetDurableExecutionHistory cannot see
+ # an in-flight child at all.
+ ("CHAINED_INVOKE", "START"): (
+ "ChainedInvokeStarted", "ChainedInvokeStartedDetails",
+ {"FunctionName": upd.get("ChainedInvokeOptions", {}).get("FunctionName", ""),
+ "Input": {"Payload": payload or "", "Truncated": False},
+ "DurableExecutionArn": rec.get("DurableExecutionArn", "")}),
("CONTEXT", "START"): ("ContextStarted", "ContextStartedDetails", {}),
("CONTEXT", "SUCCEED"): ("ContextSucceeded", "ContextSucceededDetails",
{"Result": {"Payload": payload or "", "Truncated": False}}),
@@ -714,6 +789,11 @@ def _apply_update(rec: dict, upd: dict) -> None:
name=name, parent_id=parent_id, sub_type=sub_type,
event_id=op_id)
+ # Kick off the chained child only after its Started event is on the log.
+ if op_type == "CHAINED_INVOKE" and action == "START" and (
+ upd.get("ChainedInvokeOptions") or {}).get("FunctionName"):
+ _fire_chained_invoke(rec, op_id, upd.get("ChainedInvokeOptions") or {}, payload)
+
def handle_get_state(arn_path: str, query_params: dict) -> tuple:
rec, err = _require_execution(arn_path)
@@ -905,6 +985,20 @@ def handle_stop(arn_path: str, body: bytes) -> tuple:
"ErrorData": data.get("ErrorData") or "",
"StackTrace": data.get("StackTrace") or [],
}
+ # A stop reaches the in-flight chained invokes too: each STARTED
+ # CHAINED_INVOKE op lands STOPPED with its own history event
+ # (ChainedInvokeStopped is a real EventType on the model), and the late
+ # child result is discarded by the RUNNING guard in
+ # _append_chained_result instead of appending onto a stopped execution.
+ for op in rec.get("Operations", []):
+ if op.get("Type") == "CHAINED_INVOKE" and op.get("Status") == "STARTED":
+ op["Status"] = "STOPPED"
+ op["EndTimestamp"] = _now()
+ op.setdefault("ChainedInvokeDetails", {})["Error"] = rec["Error"]
+ _emit_history_event(rec, "ChainedInvokeStopped",
+ "ChainedInvokeStoppedDetails",
+ {"Error": {"Payload": rec["Error"], "Truncated": False}},
+ event_id=op.get("Id"))
_emit_history_event(rec, "ExecutionStopped", "ExecutionStoppedDetails", {
"Error": {"Payload": rec["Error"], "Truncated": False},
})
@@ -1169,6 +1263,12 @@ def schedule_resume(arn: str, account_id: str | None = None,
if not rec or rec.get("Status") != "RUNNING":
return False
expiry = _next_expiry(rec)
+ deadline = rec.get("_TimeoutDeadline")
+ if deadline is not None:
+ # The execution-timeout deadline is a wake-up like any other: the
+ # earliest pending timer wins, and an execution with no waits at all
+ # still gets its timeout enforced.
+ expiry = deadline if expiry is None else min(expiry, deadline)
# When called from a Send*Callback handler we want to wake the execution
# immediately so the SDK observes the resolution on its next replay.
has_resolved_callback = any(
@@ -1252,6 +1352,13 @@ def _resume_execution(arn: str, account_id: str = "000000000000",
if not rec or rec.get("Status") != "RUNNING":
return
now = _now()
+ deadline = rec.get("_TimeoutDeadline")
+ if deadline is not None and now >= deadline:
+ # DurableConfig.ExecutionTimeout elapsed: the execution lands
+ # TIMED_OUT instead of being re-invoked, and every in-flight chained
+ # invoke is timed out with it.
+ _time_out_execution(rec)
+ return
anything_elapsed = False
has_resolved_callback = any(
op.get("Type") == "CALLBACK" and op.get("Status") in ("SUCCEEDED", "FAILED")
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index be78e8b5..9cbf85ca 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -1261,7 +1261,14 @@ def invoke_durable_resume(function_name: str, durable_arn: str, original_event:
},
}
try:
+ _replay_started = time.time()
result = _execute_function(func, resume_event)
+ lambda_durable.record_invocation_completed(
+ durable_arn,
+ start_ts=_replay_started,
+ request_id=new_uuid(),
+ error=(result.get("body") if result.get("error") else None),
+ )
# If the resume still returns PENDING, schedule the next wakeup.
try:
payload = result.get("body")
@@ -2840,6 +2847,8 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | N
function_arn=_func_arn(name),
version=executed_version,
input_payload=event_payload,
+ execution_timeout=(func.get("config", {}) or {}).get(
+ "DurableConfig", {}).get("ExecutionTimeout"),
)
durable_arn = rec["DurableExecutionArn"]
_durable_ctx.set({
@@ -2899,6 +2908,15 @@ async def _invoke(name: str, event: dict, headers: dict, path_qualifier: str | N
_de_rec = lambda_durable._executions.get(durable_arn)
if _de_rec:
resp_headers["X-Amz-Durable-Checkpoint-Token"] = _de_rec["CheckpointToken"]
+ # Every handler invocation of a durable execution ends with an
+ # InvocationCompleted history event (initial and each replay), before
+ # any terminal ExecutionSucceeded/Failed the payload may cause.
+ lambda_durable.record_invocation_completed(
+ durable_arn,
+ start_ts=(_de_rec or {}).get("StartTimestamp") or time.time(),
+ request_id=new_uuid(),
+ error=(result.get("body") if result.get("error") else None),
+ )
# Inspect the SDK's return value: PENDING → schedule the next wakeup
# from the latest WAIT timestamp; SUCCEEDED/FAILED → mark terminal.
try:
diff --git a/tests/test_lambda.py b/tests/test_lambda.py
index 68a9c7a9..3d4851db 100644
--- a/tests/test_lambda.py
+++ b/tests/test_lambda.py
@@ -8011,6 +8011,112 @@ def test_lambda_durable_stop(lam):
lam.delete_function(FunctionName=fname)
+def test_lambda_durable_invocation_completed_recorded(lam):
+ """Every handler invocation of a durable execution ends with an
+ InvocationCompleted history event (a real EventType on the model,
+ previously never emitted), carrying a RequestId and both timestamps."""
+ fname, _, rec = _create_durable_execution_directly(lam)
+ try:
+ from urllib.parse import quote
+ arn = quote(rec["DurableExecutionArn"], safe="/:$")
+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")
+ assert code == 200
+ completed = [e for e in body["Events"] if e["EventType"] == "InvocationCompleted"]
+ assert completed, [e["EventType"] for e in body["Events"]]
+ details = completed[0]["InvocationCompletedDetails"]
+ assert details["RequestId"]
+ assert details["EndTimestamp"] >= details["StartTimestamp"] - 1
+ finally:
+ lam.delete_function(FunctionName=fname)
+
+
+def test_lambda_durable_execution_timeout_enforced(lam):
+ """DurableConfig.ExecutionTimeout (a real field, seconds) caps the whole
+ execution: it was stored and ignored, so a RUNNING execution could out-
+ live its cap forever. It now lands TIMED_OUT with an ExecutionTimedOut
+ event once the deadline passes. The wait below is the 1-second timeout
+ itself firing, not a contention budget."""
+ import base64 as _b64
+ fname = f"durable-timeout-{_uuid_mod.uuid4().hex[:8]}"
+ zip_b64 = _b64.b64encode(_make_zip("def handler(e,c): return {}")).decode()
+ _raw_durable("POST", "/2015-03-31/functions", body={
+ "FunctionName": fname, "Runtime": "python3.12", "Role": _LAMBDA_ROLE,
+ "Handler": "index.handler", "Code": {"ZipFile": zip_b64},
+ "DurableConfig": {"Enabled": True, "ExecutionTimeout": 1},
+ })
+ try:
+ invoke_req = urllib.request.Request(
+ f"{_ms_endpoint()}/2015-03-31/functions/{fname}/invocations",
+ method="POST", data=b"{}", headers={"Content-Type": "application/json"},
+ )
+ with urllib.request.urlopen(invoke_req) as r:
+ arn_raw = r.headers.get("X-Amz-Durable-Execution-Arn")
+ r.read()
+ from urllib.parse import quote
+ arn = quote(arn_raw, safe="/:$")
+ deadline = time.time() + 10
+ status = None
+ while time.time() < deadline:
+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}")
+ status = body["Status"]
+ if status != "RUNNING":
+ break
+ time.sleep(0.2)
+ assert status == "TIMED_OUT", status
+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")
+ assert any(e["EventType"] == "ExecutionTimedOut" for e in hist["Events"]), (
+ [e["EventType"] for e in hist["Events"]])
+ finally:
+ lam.delete_function(FunctionName=fname)
+
+
+def test_lambda_durable_stop_reaches_inflight_chained_invoke(lam):
+ """StopDurableExecution reaches an in-flight chained invoke: the op lands
+ STOPPED with a ChainedInvokeStopped event (a real EventType, previously
+ never emitted), and the child's late result is discarded instead of
+ appending ChainedInvokeSucceeded onto a stopped execution."""
+ import base64 as _b64
+ child = f"durable-stopchild-{_uuid_mod.uuid4().hex[:8]}"
+ _raw_durable("POST", "/2015-03-31/functions", body={
+ "FunctionName": child, "Runtime": "python3.12", "Role": _LAMBDA_ROLE,
+ "Handler": "index.handler",
+ "Code": {"ZipFile": _b64.b64encode(_make_zip(
+ "import time\ndef handler(e,c):\n time.sleep(2)\n return {'late': True}")).decode()},
+ "Timeout": 30,
+ })
+ fname, _, rec = _create_durable_execution_directly(lam)
+ try:
+ from urllib.parse import quote
+ arn = quote(rec["DurableExecutionArn"], safe="/:$")
+ code, _b = _raw_durable("POST", f"/2025-12-01/durable-executions/{arn}/checkpoint", body={
+ "CheckpointToken": rec["CheckpointToken"],
+ "Updates": [{"Id": "chain-stop", "Type": "CHAINED_INVOKE",
+ "Action": "START", "Name": "slow-child",
+ "ChainedInvokeOptions": {"FunctionName": child}}],
+ })
+ assert code == 200
+ code, _b = _raw_durable("POST", f"/2025-12-01/durable-executions/{arn}/stop", body={
+ "ErrorMessage": "stopping with child in flight",
+ })
+ assert code == 200
+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")
+ types = [e["EventType"] for e in hist["Events"]]
+ assert "ChainedInvokeStarted" in types, types
+ assert "ChainedInvokeStopped" in types, types
+ # Past the child's 2s sleep: its late result must not have been applied.
+ time.sleep(3)
+ code, hist = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}/history")
+ types = [e["EventType"] for e in hist["Events"]]
+ assert "ChainedInvokeSucceeded" not in types, types
+ code, body = _raw_durable("GET", f"/2025-12-01/durable-executions/{arn}")
+ assert body["Status"] == "STOPPED"
+ finally:
+ lam.delete_function(FunctionName=fname)
+ lam.delete_function(FunctionName=child)
+
+
def test_lambda_durable_list_by_function(lam):
fname, fn_arn, rec = _create_durable_execution_directly(lam)
try:
@@ -8078,6 +8184,7 @@ def handler(event, context):
pass
def test_lambda_durable_chained_invoke_runs_child(lam):
"""A CHAINED_INVOKE checkpoint update with Action=START actually spawns
the child function and records the result back into the parent's
@@ -8133,13 +8240,24 @@ def test_lambda_durable_chained_invoke_runs_child(lam):
}],
})
assert code == 200
- # The child runs in a daemon thread; give it a moment.
+ # ChainedInvokeStarted is recorded synchronously when the checkpoint
+ # accepts the START update, so it must already be on the log — no
+ # polling, no budget.
+ code, history = _raw_durable("GET",
+ f"/2025-12-01/durable-executions/{arn_enc}/history")
+ started = [e for e in history["Events"] if e["EventType"] == "ChainedInvokeStarted"]
+ assert started, [e["EventType"] for e in history["Events"]]
+ assert started[0]["ChainedInvokeStartedDetails"]["FunctionName"] == child
+ # The child completes in a background thread; serial keeps the box
+ # quiet so the short poll is honest. Breaking on Failed keeps a real
+ # child error visible instead of reading as a timeout.
import time as _time
for _ in range(30):
_time.sleep(0.1)
code, history = _raw_durable("GET",
f"/2025-12-01/durable-executions/{arn_enc}/history")
- if any(e["EventType"] == "ChainedInvokeSucceeded" for e in history.get("Events", [])):
+ if any(e["EventType"] in ("ChainedInvokeSucceeded", "ChainedInvokeFailed")
+ for e in history.get("Events", [])):
break
events = history["Events"]
assert any(e["EventType"] == "ChainedInvokeSucceeded" for e in events), \