Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(lambda): preserve invocation IDs and avoid duplicate RIE logs (#2086)
gakuto-cw2123e0536eee17
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 60aa5154..0084d0cf 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -29,6 +29,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).### Fixed+- **Lambda — invocation IDs and Docker logs** — one request ID is propagated to Docker RIE, warm/local Python and Node.js, and provided runtimes. RIE output is identified by executor metadata and saved without a second START / END / REPORT sequence. Other executors and empty RIE output retain synthetic platform logs.- **Lambda — event source mappings pick up new work at once** — a new SQS message, Kinesis record or DynamoDB stream record now wakes the poller instead of waiting for its idle tick, cutting the delay from send to invocation from 0.75 s to about 4 ms.- **TLS — the generated certificate works with rustls** — it was a CA certificate, which rustls refuses as a server's (`CaUsedAsEndEntity`). It is now a server certificate (`CA:FALSE`, `serverAuth`), and a cached CA certificate is regenerated.- **Lambda — preserve invocation aliases in runtime context** — `Invoke` retains the requested alias or version in `invoked_function_arn`, while `function_version` remains the resolved published version. Warm workers receive invocation identity per request. Contributed by @jayjanssen.diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.pyindex a0587aaf..b1dc212a 100644--- a/ministack/services/lambda_svc.py+++ b/ministack/services/lambda_svc.py@@ -807,9 +807,9 @@ class LambdaContext:function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "")memory_limit_in_mb = int(os.environ.get("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "128"))invoked_function_arn = os.environ.get("_LAMBDA_FUNCTION_ARN", "")- aws_request_id = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME", "")+ aws_request_id = os.environ["_LAMBDA_REQUEST_ID"]log_group_name = "/aws/lambda/" + function_name- log_stream_name = aws_request_id+ log_stream_name = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME", "")@staticmethoddef get_remaining_time_in_millis():@@ -904,7 +904,7 @@ const context = {functionName: process.env.AWS_LAMBDA_FUNCTION_NAME || '',memoryLimitInMB: process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE || '128',invokedFunctionArn: process.env._LAMBDA_FUNCTION_ARN || '',- awsRequestId: process.env.AWS_LAMBDA_LOG_STREAM_NAME || '',+ awsRequestId: process.env._LAMBDA_REQUEST_ID || '',logGroupName: '/aws/lambda/' + (process.env.AWS_LAMBDA_FUNCTION_NAME || ''),logStreamName: process.env.AWS_LAMBDA_LOG_STREAM_NAME || '',getRemainingTimeInMillis: () => parseFloat(process.env._LAMBDA_TIMEOUT || '3') * 1000,@@ -3972,7 +3972,7 @@ def _rie_terminal_result(exc: BaseException, timeout: int, logs: str) -> dict:}-def _invoke_rie(container, event: dict, timeout: int) -> dict:+def _invoke_rie(container, event: dict, timeout: int, request_id: str) -> dict:"""POST event to a running RIE container's HTTP endpoint."""import urllib.request# A CloudFormation custom-resource ResponseURL points at ministack on the@@ -4015,7 +4015,7 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:invoke_time = time.time()req = urllib.request.Request(rie_url, data=json.dumps(event).encode(),- headers={"Content-Type": "application/json"},+ headers={"Content-Type": "application/json", "X-Amzn-RequestId": request_id},)resp = urllib.request.urlopen(req, timeout=timeout)body = resp.read().decode("utf-8", errors="replace")@@ -4026,7 +4026,9 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:logs = container.logs(stdout=True, stderr=True, since=invoke_time).decode("utf-8", errors="replace").strip()err_header = (resp.headers.get("X-Amz-Function-Error")or resp.headers.get("Lambda-Runtime-Function-Error-Type") or "")- result = {"body": parsed, "log": logs}+ # RIE owns the platform lines, including handler errors returned as HTTP 200.+ # Carry provenance separately so user output never controls log wrapping.+ result = {"body": parsed, "log": logs, "log_source": "rie"}function_error = _classify_function_error(parsed, err_header)if function_error is not None:result["error"] = True@@ -4041,7 +4043,9 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:logs = container.logs(stdout=True, stderr=True, since=attempt_start).decode("utf-8", errors="replace").strip()- return _rie_terminal_result(exc, timeout, logs)+ result = _rie_terminal_result(exc, timeout, logs)+ result["log_source"] = "rie"+ return resulttime.sleep(0.1)continue# Never became reachable@@ -4693,7 +4697,7 @@ def _spawn_lambda_container_impl(config: dict, code_zip: bytes | None,return container, tmpdir-def _execute_function_docker(func: dict, event: dict) -> dict:+def _execute_function_docker(func: dict, event: dict, request_id: str) -> dict:"""Execute a Lambda function inside a Docker container using AWS RIE.Unifies Zip and Image PackageType through a single warm container pool.@@ -4715,9 +4719,9 @@ def _execute_function_docker(func: dict, event: dict) -> dict:"errorType": "Runtime.DockerUnavailable"}, "error": True}if runtime.startswith(("python", "nodejs")):logger.warning("docker SDK unavailable - falling back to warm executor")- return _execute_function_warm(func, event)+ return _execute_function_warm(func, event, request_id)logger.warning("docker SDK unavailable - falling back to local subprocess")- return _execute_function_local(func, event)+ return _execute_function_local(func, event, request_id)if _get_docker_client() is None:if package_type == "Image" or LAMBDA_STRICT:@@ -4725,8 +4729,8 @@ def _execute_function_docker(func: dict, event: dict) -> dict:"errorType": "Runtime.DockerError"}, "error": True}logger.warning("Docker daemon unreachable – falling back")if runtime.startswith(("python", "nodejs")):- return _execute_function_warm(func, event)- return _execute_function_local(func, event)+ return _execute_function_warm(func, event, request_id)+ return _execute_function_local(func, event, request_id)# Zip needs code (Image brings it in the image)code_zip = func.get("code_zip")@@ -4787,7 +4791,7 @@ def _execute_function_docker(func: dict, event: dict) -> dict:time.sleep(0.05)try:- result = _invoke_rie(entry["container"], event, timeout)+ result = _invoke_rie(entry["container"], event, timeout, request_id)# `timeout` is _invoke_rie's internal signal to this function; pop it so# only the caller-facing keys survive into the invoke response.if result.pop("timeout", False):@@ -4873,12 +4877,13 @@ def _probe_peak_memory_mb(func: dict) -> int:def _emit_lambda_logs(func: dict, request_id: str, log_text: str,- error: bool, duration_ms: int) -> None:+ error: bool, duration_ms: int, *, log_source: str | None = None) -> None:"""Write handler output to CloudWatch Logs under /aws/lambda/{name}, matching AWS.- Log group `/aws/lambda/{FunctionName}` is auto-created on first write.- Stream name follows AWS's format: `{yyyy}/{mm}/{dd}/[{qualifier}]{uuid}`.- - Each invocation emits START / body / END / REPORT lines like real Lambda.+ - RIE output already contains platform lines; other executors get a wrapper.+ - Empty RIE output falls back to the synthetic wrapper so logs remain visible.Best-effort: a failure to write logs must never break the invocation."""try:@@ -4916,16 +4921,19 @@ def _emit_lambda_logs(func: dict, request_id: str, log_text: str,}stream = group["streams"][stream_name]- lines: list[str] = [f"START RequestId: {request_id} Version: {qualifier}"]- if log_text:- lines.extend(log_text.splitlines())- lines.append(f"END RequestId: {request_id}")- peak_mb = _probe_peak_memory_mb(func)- lines.append(- f"REPORT RequestId: {request_id}\tDuration: {duration_ms} ms\t"- f"Billed Duration: {duration_ms} ms\tMemory Size: "- f"{config.get('MemorySize', 128)} MB\tMax Memory Used: {peak_mb} MB"- )+ if log_source == "rie" and log_text:+ lines = log_text.splitlines()+ else:+ lines = [f"START RequestId: {request_id} Version: {qualifier}"]+ if log_text:+ lines.extend(log_text.splitlines())+ lines.append(f"END RequestId: {request_id}")+ peak_mb = _probe_peak_memory_mb(func)+ lines.append(+ f"REPORT RequestId: {request_id}\tDuration: {duration_ms} ms\t"+ f"Billed Duration: {duration_ms} ms\tMemory Size: "+ f"{config.get('MemorySize', 128)} MB\tMax Memory Used: {peak_mb} MB"+ )for line in lines:stream["events"].append({"timestamp": now_ms, "message": line, "ingestionTime": now_ms})if stream["firstEventTimestamp"] is None:@@ -5040,6 +5048,7 @@ def _execute_function(func: dict, event: dict) -> dict:# the slot for a single invocation.return _run_with_function_config_scope(config, _execute_function, func, event)+ # One invocation identity for the context, Runtime API and platform logs.request_id = new_uuid()started = time.time()@@ -5081,11 +5090,11 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,if proxy_url:result = _execute_function_proxy(func, event, proxy_url, request_id)elif LAMBDA_STRICT:- result = _execute_function_docker(func, event)+ result = _execute_function_docker(func, event, request_id)elif config.get("PackageType") == "Image" and config.get("ImageUri"):- result = _execute_function_docker(func, event)+ result = _execute_function_docker(func, event, request_id)elif LAMBDA_EXECUTOR == "docker":- result = _execute_function_docker(func, event)+ result = _execute_function_docker(func, event, request_id)else:runtime = config.get("Runtime", "python3.12")if runtime.startswith("provided"):@@ -5095,17 +5104,17 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,# invocations keep the one-shot executor. python and nodejs carry# that context in the event instead, which is why they can be pooled.if _durable_ctx.get():- result = _execute_function_provided(func, event)+ result = _execute_function_provided(func, event, request_id)else:result = _execute_function_provided_warm(func, event, request_id)elif runtime.startswith(("python", "nodejs")):# Durable invocations included: their per-call context rides in# the event, so the pooled worker can serve them.- result = _execute_function_warm(func, event)+ result = _execute_function_warm(func, event, request_id)else:# java*/dotnet*/ruby* need the real RIE image — there's no# in-process executor that can run JVM bytecode or .NET IL.- result = _execute_function_docker(func, event)+ result = _execute_function_docker(func, event, request_id)duration_ms = int((time.time() - started) * 1000)_emit_lambda_logs(@@ -5113,6 +5122,7 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,result.get("log", "") if isinstance(result, dict) else "",bool(result.get("error")) if isinstance(result, dict) else False,duration_ms,+ log_source=result.get("log_source") if isinstance(result, dict) else None,)return result@@ -5238,7 +5248,7 @@ def _execute_function_proxy(func: dict, event: dict, url: str, request_id: str)return {"body": text}-def _execute_function_warm(func: dict, event: dict) -> dict:+def _execute_function_warm(func: dict, event: dict, request_id: str) -> dict:"""Execute a Lambda function using the warm worker pool (Python + Node.js)."""config = func.get("config") or funccode_zip = func.get("code_zip")@@ -5274,7 +5284,7 @@ def _execute_function_warm(func: dict, event: dict) -> dict:# the values ride the envelope and the bootstrap applies them per call.result = worker.invoke(event,- new_uuid(),+ request_id,trace_id=_xray_trace_id_for_invocation(config),invoked_function_arn=config.get("FunctionArn", ""),depth=_invoke_depth.get(),@@ -5348,7 +5358,7 @@ def _provided_worker_env(config: dict, code_dir: str, port: int) -> dict:def _execute_function_provided_warm(func: dict, event: dict,- request_id: str | None = None) -> dict:+ request_id: str) -> dict:"""Execute a ``provided.*`` Lambda on a pooled, reused environment."""config = func.get("config") or funccode_zip = func.get("code_zip")@@ -5368,7 +5378,7 @@ def _execute_function_provided_warm(func: dict, event: dict,# Invocation metadata belongs in Runtime API headers, not in the user# payload (which need not be a dict and may contain similarly named keys).result = worker.invoke(- event, request_id or new_uuid(),+ event, request_id,trace_id=_xray_trace_id_for_invocation(config),invoked_function_arn=config.get("FunctionArn", ""),)@@ -5405,7 +5415,7 @@ def _execute_function_provided_warm(func: dict, event: dict,release_worker(worker)-def _execute_function_provided(func: dict, event: dict) -> dict:+def _execute_function_provided(func: dict, event: dict, request_id: str) -> dict:"""Execute a provided-runtime Lambda (Go/Rust binary) via a minimal Lambda Runtime API."""config = func.get("config") or funccode_zip = func.get("code_zip")@@ -5425,7 +5435,6 @@ def _execute_function_provided(func: dict, event: dict) -> dict:# Shared state for the Runtime APIresult_holder = {"response": None, "error": None}event_json = json.dumps(event)- request_id = new_uuid()event_served = threading.Event()response_received = threading.Event()server_ready = threading.Event()@@ -5565,7 +5574,7 @@ def _execute_function_provided(func: dict, event: dict) -> dict:return {"body": {"errorMessage": str(e), "errorType": type(e).__name__}, "error": True}-def _execute_function_local(func: dict, event: dict) -> dict:+def _execute_function_local(func: dict, event: dict, request_id: str) -> dict:"""Execute a Lambda function in a one-shot subprocess (fallback for unsupported runtimes)."""config = func.get("config") or funccode_zip = func.get("code_zip")@@ -5685,6 +5694,7 @@ def _execute_function_local(func: dict, event: dict) -> dict:if _xray_trace_id:env["_X_AMZN_TRACE_ID"] = _xray_trace_idenv[INVOKE_DEPTH_ENV] = str(_invoke_depth.get())+ env["_LAMBDA_REQUEST_ID"] = request_idcmd = ["node", wrapper_path] if is_node else ["python3", wrapper_path]proc = subprocess.run(diff --git a/tests/test_lambda.py b/tests/test_lambda.pyindex df8348da..b8e94fb7 100644--- a/tests/test_lambda.py+++ b/tests/test_lambda.py@@ -4099,7 +4099,7 @@ def test_lambda_provided_runtime_env_has_function_vars():},"code_zip": buf.getvalue(),}- result = lmod._execute_function_provided(func, {"ping": "pong"})+ result = lmod._execute_function_provided(func, {"ping": "pong"}, request_id="request-1")assert not result.get("error"), resultbody = result["body"]assert body["memory"] == "512"@@ -4150,7 +4150,7 @@ def test_lambda_provided_runtime_parallel_invocations():with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:results = list(ex.map(- lambda i: lmod._execute_function_provided(func, {"n": i}), range(8)))+ lambda i: lmod._execute_function_provided(func, {"n": i}, request_id=f"request-{i}"), range(8)))for i, r in enumerate(results):assert not r.get("error"), f"invocation {i} failed: {r}"@@ -5352,7 +5352,7 @@ def test_execute_function_uses_function_config_region_for_logs(monkeypatch):monkeypatch.setattr(lsvc,"_execute_function_warm",- lambda _func, _event: {"body": {"ok": True}, "log": "ran in target region"},+ lambda _func, _event, request_id: {"body": {"ok": True}, "log": "ran in target region"},)cwl.reset()try:@@ -7011,7 +7011,7 @@ def test_warm_invocation_leaves_the_caller_event_untouched(monkeypatch):event = {"input": "hi"}token = lsvc._durable_ctx.set({"arn": "exec-arn", "token": "tok", "name": "exec"})try:- result = lsvc._execute_function_warm({"config": config, "code_zip": b"zip"}, event)+ result = lsvc._execute_function_warm({"config": config, "code_zip": b"zip"}, event, request_id="request-1")finally:lsvc._durable_ctx.reset(token)@@ -7123,7 +7123,7 @@ def test_lambda_strict_hard_fails_when_docker_unavailable(monkeypatch):"Timeout": 3,"MemorySize": 128,}, "code_zip": b"\x00"}- result = lsvc._execute_function_docker(func, {"k": "v"})+ result = lsvc._execute_function_docker(func, {"k": "v"}, request_id="request-1")assert result.get("error") is Trueassert result["body"]["errorType"] == "Runtime.DockerUnavailable"@@ -7133,7 +7133,7 @@ def test_lambda_permissive_falls_back_to_warm_without_docker(monkeypatch):monkeypatch.setattr(lsvc, "LAMBDA_STRICT", False)monkeypatch.setattr(lsvc, "_docker_available", False)called = {"warm": False}- def _fake_warm(func, event):+ def _fake_warm(func, event, request_id):called["warm"] = Truereturn {"body": {"ok": True}}monkeypatch.setattr(lsvc, "_execute_function_warm", _fake_warm)@@ -7145,7 +7145,7 @@ def test_lambda_permissive_falls_back_to_warm_without_docker(monkeypatch):"Timeout": 3,"MemorySize": 128,}, "code_zip": b"\x00"}- lsvc._execute_function_docker(func, {})+ lsvc._execute_function_docker(func, {}, request_id="request-1")assert called["warm"] is True@@ -10584,7 +10584,7 @@ def test_invoke_rie_rewrites_custom_resource_response_url():"ResourceProperties": {},}with patch("urllib.request.urlopen", _fake_urlopen):- _invoke_rie(_FakeContainer(), event, timeout=5)+ _invoke_rie(_FakeContainer(), event, timeout=5, request_id="request-1")assert captured["body"]["ResponseURL"] == \"http://host.docker.internal:4566/_ministack/cfn-response/tok"@@ -10666,7 +10666,7 @@ def test_invoke_rie_reports_an_init_error_instead_of_retrying_it():started = time.time()with patch("urllib.request.urlopen", _fake_urlopen):# Timeout=900 is legal on AWS, and used to buy 9020 retries at 0.1s.- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")elapsed = time.time() - startedassert len(calls) == 1@@ -10696,7 +10696,7 @@ def test_invoke_rie_still_retries_a_container_that_is_still_starting():return _FakeResp()with patch("urllib.request.urlopen", _fake_urlopen):- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=5)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=5, request_id="request-1")assert len(calls) == 3assert result["body"] == {"ok": True}@@ -10714,7 +10714,7 @@ def test_invoke_rie_does_not_re_run_a_handler_that_timed_out():raise TimeoutError("timed out")with patch("urllib.request.urlopen", _fake_urlopen):- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")assert len(calls) == 1assert result["error"] is True@@ -10745,7 +10745,7 @@ def test_invoke_rie_reports_a_non_json_502_body():)with patch("urllib.request.urlopen", _fake_urlopen):- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")assert result["error"] is Trueassert result["function_error"] == "Unhandled"@@ -10766,7 +10766,7 @@ def test_invoke_rie_non_json_502_without_an_error_type_header():)with patch("urllib.request.urlopen", _fake_urlopen):- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")assert result["body"] == {"errorMessage": "Lambda RIE returned HTTP 502: not json",@@ -10812,10 +10812,10 @@ def test_lambda_docker_timeout_recycles_the_container(monkeypatch):try:with patch("urllib.request.urlopen", _fake_urlopen):first = lsvc._execute_function_docker(- {"config": config, "code_zip": b"zip"}, {}+ {"config": config, "code_zip": b"zip"}, {}, "request-1",)second = lsvc._execute_function_docker(- {"config": config, "code_zip": b"zip"}, {}+ {"config": config, "code_zip": b"zip"}, {}, "request-1",)finally:lsvc._pool_kill_function("000000000000", name)@@ -10848,7 +10848,7 @@ def test_invoke_rie_gives_up_on_an_unreachable_container(monkeypatch):started = time.time()with patch("urllib.request.urlopen", _fake_urlopen):- result = lsvc._invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)+ result = lsvc._invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")elapsed = time.time() - startedassert elapsed < 5@@ -13756,7 +13756,7 @@ def test_provided_failure_is_scoped_and_not_retried(monkeypatch, isolated_pool):unrelated.append(worker)fallback = Mock()monkeypatch.setattr(lambda_svc, "_execute_function_provided", fallback)- result = lambda_svc._execute_function_provided_warm({"config": config, "code_zip": b"zip"}, {})+ result = lambda_svc._execute_function_provided_warm({"config": config, "code_zip": b"zip"}, {}, request_id="request-1")assert result["error"] is Trueassert result["body"]["errorMessage"] == "bootstrap crashed"assert fail.call_count == 1@@ -13864,11 +13864,11 @@ def test_docker_unavailable_fallback_carries_the_durable_context(monkeypatch, unfunc = {"config": config, "code_zip": b"zip"}token = lambda_svc._durable_ctx.set({"arn": "exec-arn", "token": "tok", "name": "exec"})try:- assert lambda_svc._execute_function_docker(func, {}) == {"body": "warm"}+ assert lambda_svc._execute_function_docker(func, {}, request_id="request-1") == {"body": "warm"}overlay = lambda_svc._durable_env_overlay()finally:lambda_svc._durable_ctx.reset(token)- warm.assert_called_once_with(func, {})+ warm.assert_called_once_with(func, {}, "request-1")one_shot.assert_not_called()assert overlay == {"AWS_LAMBDA_DURABLE_EXECUTION_ARN": "exec-arn",@@ -14251,7 +14251,7 @@ def test_invoke_rie_reports_a_bare_string_timeout_as_a_function_error():return b"Task timed out after 300.00 seconds"with patch("urllib.request.urlopen", lambda req, timeout=None: _FakeResp()):- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")assert result["error"] is Trueassert result["function_error"] == "Unhandled"@@ -14356,9 +14356,12 @@ def test_invocation_retains_alias_without_mutating_version(monkeypatch, asynchrooriginal = deepcopy(function)monkeypatch.setitem(svc._functions, name, function)monkeypatch.setattr(svc, "_emit_lambda_metrics", lambda *args, **kwargs: None)- monkeypatch.setattr(svc, "_execute_function_with_config_scope", svc._execute_function_warm)+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "local")+ monkeypatch.setattr(svc, "LAMBDA_STRICT", False)+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: None)+ monkeypatch.setattr(svc, "_emit_lambda_logs", lambda *args, **kwargs: None)captured = []- monkeypatch.setattr(svc, "invoke_async_with_retry", lambda f, e: captured.append(svc._execute_function_warm(f, e)))+ monkeypatch.setattr(svc, "invoke_async_with_retry", lambda f, e: captured.append(svc._execute_function_with_config_scope(f, e)))try:# Alternate aliases sharing one published version, including warm reuse.for qualifier in ["worker", "other", "1", "worker", None, "$LATEST"]:diff --git a/tests/test_lambda_invocation_logs.py b/tests/test_lambda_invocation_logs.pynew file mode 100644index 00000000..2a7def91--- /dev/null+++ b/tests/test_lambda_invocation_logs.py@@ -0,0 +1,322 @@+"""Invocation IDs and platform logs: AWS docs and local RIE observations.++Docker tests use real RIE containers; no live AWS account validation is claimed.+"""+import asyncio+import base64+import io+import json+import re+import shutil+import time+import urllib.error+import uuid+import zipfile+from unittest.mock import Mock++import pytest++from ministack.core import lambda_runtime+from ministack.core.responses import AccountRegionScopedDict+from ministack.services import cloudwatch_logs as cwl+from ministack.services import lambda_svc as svc+++def log_state(monkeypatch):+ monkeypatch.setattr(cwl, "_log_groups", AccountRegionScopedDict())+ monkeypatch.setattr(cwl, "_fanout_to_subscription_filters", Mock())+ monkeypatch.setattr(svc, "_probe_peak_memory_mb", lambda func: 12)+ monkeypatch.setattr(svc, "_ensure_reaper_thread", lambda: None)+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: None)+ monkeypatch.setattr(svc, "LAMBDA_STRICT", False)+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "local")+++def _function(runtime="python3.12", code_zip=None):+ name = "invocation-logs-" + uuid.uuid4().hex+ return {"config": {+ "FunctionName": name,+ "FunctionArn": svc._func_arn(name),+ "Runtime": runtime, "Handler": "index.handler", "Version": "$LATEST",+ "MemorySize": 128, "Timeout": 10, "PackageType": "Zip",+ }, "code_zip": code_zip}+++def _messages(func):+ group = cwl._log_groups["/aws/lambda/" + func["config"]["FunctionName"]]+ return [e["message"] for s in group["streams"].values() for e in s["events"]]+++def _assert_platform_lines(messages, request_id):+ for kind in ("START", "END", "REPORT"):+ ids = [m.group(1) for line in messages+ if (m := re.match(rf"^{kind} RequestId: (\S+)", line))]+ assert ids == [request_id]++[email protected]("mode,target", [+ ("python", "_execute_function_warm"), ("node", "_execute_function_warm"),+ ("docker", "_execute_function_docker"), ("strict", "_execute_function_docker"),+ ("image", "_execute_function_docker"), ("other", "_execute_function_docker"),+ ("provided", "_execute_function_provided_warm"),+ ("durable", "_execute_function_provided"), ("proxy", "_execute_function_proxy"),+])+def test_execute_function_generates_one_request_id(monkeypatch, log_state, mode, target):+ func = _function()+ if mode == "node":+ func["config"]["Runtime"] = "nodejs22.x"+ elif mode == "other":+ func["config"]["Runtime"] = "ruby3.3"+ elif mode in ("provided", "durable"):+ func["config"]["Runtime"] = "provided.al2023"+ elif mode == "image":+ func["config"].update(PackageType="Image", ImageUri="example:latest")+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker" if mode == "docker" else "local")+ monkeypatch.setattr(svc, "LAMBDA_STRICT", mode == "strict")+ if mode == "proxy":+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: "http://proxy")+ generated = Mock(return_value="invocation-id")+ executor = Mock(return_value={"body": {"ok": True}, "log": "user output"})+ emit = Mock()+ monkeypatch.setattr(svc, "new_uuid", generated)+ monkeypatch.setattr(svc, target, executor)+ monkeypatch.setattr(svc, "_emit_lambda_logs", emit)+ token = svc._durable_ctx.set({"arn": "execution"} if mode == "durable" else None)+ try:+ svc._execute_function(func, {"input": 1})+ finally:+ svc._durable_ctx.reset(token)+ generated.assert_called_once_with()+ args = (func, {"input": 1}, "http://proxy", "invocation-id") if mode == "proxy" else (+ func, {"input": 1}, "invocation-id")+ executor.assert_called_once_with(*args)+ assert emit.call_args.args[1] == "invocation-id"++[email protected]("unavailable", ["sdk", "daemon"])[email protected]("runtime,target", [+ ("python3.12", "_execute_function_warm"),+ ("nodejs22.x", "_execute_function_warm"),+ ("ruby3.3", "_execute_function_local"),+])+def test_docker_fallback_preserves_id_and_synthetic_logs(+ monkeypatch, log_state, unavailable, runtime, target):+ func = _function(runtime, b"zip")+ monkeypatch.setattr(svc, "_docker_available", unavailable != "sdk")+ monkeypatch.setattr(svc, "_get_docker_client", lambda: None)+ executor = Mock(return_value={"body": {}, "log": "USER_PRINT fallback"})+ monkeypatch.setattr(svc, target, executor)+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")+ svc._execute_function_dispatch(func, func["config"], {}, "same-id", time.time())+ executor.assert_called_once_with(func, {}, "same-id")+ _assert_platform_lines(_messages(func), "same-id")++[email protected]("log_source,error", [(None, False), ("rie", False), ("rie", True)])+def test_invoke_preserves_log_provenance_and_tail(monkeypatch, log_state, log_source, error):+ func = _function()+ raw = "START RequestId: same-id Version: $LATEST\nUSER_PRINT once\nEND RequestId: same-id\nREPORT RequestId: same-id\tDuration: 1.25 ms"+ monkeypatch.setitem(svc._functions, func["config"]["FunctionName"], func)+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")+ monkeypatch.setattr(svc, "_execute_function_docker", Mock(return_value={+ "body": {}, "log": raw, "log_source": log_source, "error": error}))+ monkeypatch.setattr(svc, "_emit_lambda_metrics", lambda *args, **kwargs: None)+ monkeypatch.setattr(svc, "new_uuid", lambda: "same-id")+ status, headers, _ = asyncio.run(svc._invoke(func["config"]["FunctionName"], {}, {"x-amz-log-type": "Tail"}))+ assert status == 200+ assert ("X-Amz-Function-Error" in headers) == error+ tail = base64.b64decode(headers["X-Amz-Log-Result"]).decode().splitlines()+ assert tail == raw.splitlines()+ messages = _messages(func)+ # User-written platform-looking lines remain user logs without RIE metadata.+ assert (messages if log_source == "rie" else messages[1:-2]) == tail+ if log_source == "rie":+ _assert_platform_lines(messages, "same-id")+ else:+ _assert_platform_lines([messages[0], *messages[-2:]], "same-id")+ assert len(messages) == len(tail) + 3+ forwarded = cwl._fanout_to_subscription_filters.call_args.args[2]+ assert [e["message"] for e in forwarded] == messages++[email protected]("raw", ["", "partial runtime output"])+def test_rie_empty_and_partial_logs_are_handled_without_text_parsing(log_state, raw):+ func = _function()+ svc._emit_lambda_logs(func, "same-id", raw, True, 1, log_source="rie")+ if raw:+ assert _messages(func) == [raw]+ else:+ _assert_platform_lines(_messages(func), "same-id")++[email protected]("outcome", ["ok", "handler-error", "init-error", "read-timeout", "not-running"])+def test_invoke_rie_passes_request_id_and_marks_actual_rie_output(monkeypatch, outcome):+ container = Mock()+ container.status = "exited" if outcome == "not-running" else "running"+ container.attrs = {"NetworkSettings": {"Networks": {}}}+ container.ports = {"8080/tcp": [{"HostPort": "12345"}]}+ container.logs.return_value = b"opaque output"+ monkeypatch.setattr(svc, "_running_in_container", lambda: False)+ monkeypatch.setattr(svc, "LAMBDA_DOCKER_NETWORK", "")+ response = Mock(headers={})+ response.read.return_value = json.dumps(+ {"errorType": "ValueError", "errorMessage": "failed"} if outcome == "handler-error" else {"ok": True}+ ).encode()+ seen = []++ def post(req, timeout):+ seen.append(req)+ if outcome == "init-error":+ raise urllib.error.HTTPError(req.full_url, 502, "Bad Gateway", {}, io.BytesIO(b"{}"))+ if outcome == "read-timeout":+ raise TimeoutError("timed out")+ return response++ monkeypatch.setattr(svc.urllib.request, "urlopen", post)+ result = svc._invoke_rie(container, {"input": 1}, 3, "same-id")+ assert result["log"] == "opaque output"+ if outcome == "not-running":+ assert not seen+ assert "log_source" not in result+ else:+ assert len(seen) == 1+ assert dict(seen[0].header_items())["X-amzn-requestid"] == "same-id"+ assert json.loads(seen[0].data) == {"input": 1}+ assert result["log_source"] == "rie"+++_PYTHON_HANDLER = """import logging+logging.basicConfig(level=logging.INFO)+logger = logging.getLogger()+logger.setLevel(logging.INFO)+def handler(event, context):+ print('USER_PRINT ' + event['marker'], flush=True)+ logger.info('USER_INFO %s', event['marker'])+ return {'request_id': context.aws_request_id, 'stream': context.log_stream_name}+"""+_NODE_HANDLER = """exports.handler = async (event, context) => {+ return {request_id: context.awsRequestId, stream: context.logStreamName};+};+"""+_PROVIDED_BOOTSTRAP = """#!/usr/bin/env python3+import json, os, urllib.request+base = 'http://' + os.environ['AWS_LAMBDA_RUNTIME_API'] + '/2018-06-01/runtime/'+while True:+ response = urllib.request.urlopen(base + 'invocation/next')+ request_id = response.headers['Lambda-Runtime-Aws-Request-Id']+ response.read()+ payload = json.dumps({'request_id': request_id}).encode()+ urllib.request.urlopen(urllib.request.Request(base + 'invocation/' + request_id + '/response', data=payload)).read()+ if os.environ.get('PROBE_ONE_SHOT'):+ break+"""+++def _code_zip():+ output = io.BytesIO()+ with zipfile.ZipFile(output, "w") as archive:+ archive.writestr("index.py", _PYTHON_HANDLER)+ archive.writestr("index.js", _NODE_HANDLER)+ bootstrap = zipfile.ZipInfo("bootstrap")+ bootstrap.external_attr = 0o100755 << 16+ archive.writestr(bootstrap, _PROVIDED_BOOTSTRAP)+ return output.getvalue()+++def _assert_two_invocations(func, *, rie=False):+ previous_id = None+ for seq in range(2):+ marker = str(seq)+ before = len(_messages(func)) if seq else 0+ result = svc._execute_function(func, {"marker": marker})+ assert not result.get("error"), result+ request_id = result["body"]["request_id"]+ assert request_id != previous_id+ if func["config"]["Runtime"].startswith(("python", "nodejs")):+ assert request_id != result["body"]["stream"]+ messages = _messages(func)[before:]+ _assert_platform_lines(messages, request_id)+ if rie:+ assert result["log_source"] == "rie"+ assert messages == result["log"].splitlines()+ if func["config"]["Runtime"].startswith("python"):+ for kind in ("USER_PRINT", "USER_INFO"):+ assert sum(kind + " " + marker in line for line in messages) == 1+ previous_id = request_id++[email protected]("runtime", ["python3.12", "nodejs22.x"])[email protected]("executor", ["warm", "local"])+def test_real_python_and_node_invocations_share_platform_request_id(monkeypatch, log_state, runtime, executor):+ if runtime.startswith("nodejs") and not shutil.which("node"):+ pytest.skip("Node.js is required")+ if executor == "local" and runtime.startswith("python") and not shutil.which("python3"):+ pytest.skip("local executor requires python3")+ if executor == "local":+ monkeypatch.setattr(svc, "_execute_function_warm", svc._execute_function_local)+ func = _function(runtime, _code_zip())+ try:+ _assert_two_invocations(func)+ finally:+ lambda_runtime.invalidate_worker(func["config"]["FunctionName"])++[email protected]("one_shot", [False, True])+def test_real_provided_runtime_receives_platform_request_id(monkeypatch, log_state, one_shot):+ if not shutil.which("python3"):+ pytest.skip("provided bootstrap requires /usr/bin/env python3")+ func = _function("provided.al2023", _code_zip())+ if one_shot:+ func["config"]["Environment"] = {"Variables": {"PROBE_ONE_SHOT": "1"}}+ token = svc._durable_ctx.set({"arn": "execution"} if one_shot else None)+ try:+ _assert_two_invocations(func)+ finally:+ svc._durable_ctx.reset(token)+ lambda_runtime.invalidate_worker(func["config"]["FunctionName"])++[email protected](scope="module")+def python_lambda_image(tmp_path_factory):+ docker = pytest.importorskip("docker")+ try:+ client = docker.from_env()+ client.ping()+ except docker.errors.DockerException as exc:+ pytest.skip(f"Docker daemon unavailable: {exc}")+ base = "public.ecr.aws/lambda/python:3.12"+ try:+ client.images.get(base)+ except docker.errors.ImageNotFound:+ client.images.pull(base)+ tag = "ministack-log-test:" + uuid.uuid4().hex+ build_dir = tmp_path_factory.mktemp("lambda-image")+ (build_dir / "Dockerfile").write_text(+ 'FROM ' + base + '\nCOPY index.py /var/task/index.py\nCMD ["index.handler"]\n', encoding="utf-8")+ (build_dir / "index.py").write_text(_PYTHON_HANDLER, encoding="utf-8")+ client.images.build(path=str(build_dir), tag=tag, rm=True)+ try:+ yield tag+ finally:+ client.images.remove(tag)+ client.close()++[email protected]_plane[email protected]("package_type", ["Zip", "Image"])+def test_real_docker_invocations_have_one_platform_log_sequence(monkeypatch, log_state, request, package_type):+ if svc._get_docker_client() is None:+ pytest.skip("Docker daemon unavailable")+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")+ monkeypatch.setattr(svc, "_KEEPALIVE_KILL_ON_RELEASE", False)+ spawn = Mock(wraps=svc._spawn_lambda_container)+ monkeypatch.setattr(svc, "_spawn_lambda_container", spawn)+ func = _function(code_zip=_code_zip())+ if package_type == "Image":+ func["config"].update(PackageType="Image", ImageUri=request.getfixturevalue("python_lambda_image"))+ try:+ _assert_two_invocations(func, rie=True)+ spawn.assert_called_once() # The second invocation must reuse the environment.+ finally:+ svc._pool_kill_function("000000000000", func["config"]["FunctionName"])