Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(lambda): preserve invocation IDs and avoid duplicate RIE logs (#2086)

gakuto-cw2123e0536eee17
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 60aa5154..0084d0cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
### Fixed
+- **Lambda — invocation IDs and Docker logs** — one request ID is propagated to Docker RIE, warm/local Python and Node.js, and provided runtimes. RIE output is identified by executor metadata and saved without a second START / END / REPORT sequence. Other executors and empty RIE output retain synthetic platform logs.
- **Lambda — event source mappings pick up new work at once** — a new SQS message, Kinesis record or DynamoDB stream record now wakes the poller instead of waiting for its idle tick, cutting the delay from send to invocation from 0.75 s to about 4 ms.
- **TLS — the generated certificate works with rustls** — it was a CA certificate, which rustls refuses as a server's (`CaUsedAsEndEntity`). It is now a server certificate (`CA:FALSE`, `serverAuth`), and a cached CA certificate is regenerated.
- **Lambda — preserve invocation aliases in runtime context** — `Invoke` retains the requested alias or version in `invoked_function_arn`, while `function_version` remains the resolved published version. Warm workers receive invocation identity per request. Contributed by @jayjanssen.
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index a0587aaf..b1dc212a 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -807,9 +807,9 @@ class LambdaContext:
function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "")
memory_limit_in_mb = int(os.environ.get("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "128"))
invoked_function_arn = os.environ.get("_LAMBDA_FUNCTION_ARN", "")
- aws_request_id = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME", "")
+ aws_request_id = os.environ["_LAMBDA_REQUEST_ID"]
log_group_name = "/aws/lambda/" + function_name
- log_stream_name = aws_request_id
+ log_stream_name = os.environ.get("AWS_LAMBDA_LOG_STREAM_NAME", "")
@staticmethod
def get_remaining_time_in_millis():
@@ -904,7 +904,7 @@ const context = {
functionName: process.env.AWS_LAMBDA_FUNCTION_NAME || '',
memoryLimitInMB: process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE || '128',
invokedFunctionArn: process.env._LAMBDA_FUNCTION_ARN || '',
- awsRequestId: process.env.AWS_LAMBDA_LOG_STREAM_NAME || '',
+ awsRequestId: process.env._LAMBDA_REQUEST_ID || '',
logGroupName: '/aws/lambda/' + (process.env.AWS_LAMBDA_FUNCTION_NAME || ''),
logStreamName: process.env.AWS_LAMBDA_LOG_STREAM_NAME || '',
getRemainingTimeInMillis: () => parseFloat(process.env._LAMBDA_TIMEOUT || '3') * 1000,
@@ -3972,7 +3972,7 @@ def _rie_terminal_result(exc: BaseException, timeout: int, logs: str) -> dict:
}
-def _invoke_rie(container, event: dict, timeout: int) -> dict:
+def _invoke_rie(container, event: dict, timeout: int, request_id: str) -> dict:
"""POST event to a running RIE container's HTTP endpoint."""
import urllib.request
# A CloudFormation custom-resource ResponseURL points at ministack on the
@@ -4015,7 +4015,7 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:
invoke_time = time.time()
req = urllib.request.Request(
rie_url, data=json.dumps(event).encode(),
- headers={"Content-Type": "application/json"},
+ headers={"Content-Type": "application/json", "X-Amzn-RequestId": request_id},
)
resp = urllib.request.urlopen(req, timeout=timeout)
body = resp.read().decode("utf-8", errors="replace")
@@ -4026,7 +4026,9 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:
logs = container.logs(stdout=True, stderr=True, since=invoke_time).decode("utf-8", errors="replace").strip()
err_header = (resp.headers.get("X-Amz-Function-Error")
or resp.headers.get("Lambda-Runtime-Function-Error-Type") or "")
- result = {"body": parsed, "log": logs}
+ # RIE owns the platform lines, including handler errors returned as HTTP 200.
+ # Carry provenance separately so user output never controls log wrapping.
+ result = {"body": parsed, "log": logs, "log_source": "rie"}
function_error = _classify_function_error(parsed, err_header)
if function_error is not None:
result["error"] = True
@@ -4041,7 +4043,9 @@ def _invoke_rie(container, event: dict, timeout: int) -> dict:
logs = container.logs(
stdout=True, stderr=True, since=attempt_start
).decode("utf-8", errors="replace").strip()
- return _rie_terminal_result(exc, timeout, logs)
+ result = _rie_terminal_result(exc, timeout, logs)
+ result["log_source"] = "rie"
+ return result
time.sleep(0.1)
continue
# Never became reachable
@@ -4693,7 +4697,7 @@ def _spawn_lambda_container_impl(config: dict, code_zip: bytes | None,
return container, tmpdir
-def _execute_function_docker(func: dict, event: dict) -> dict:
+def _execute_function_docker(func: dict, event: dict, request_id: str) -> dict:
"""Execute a Lambda function inside a Docker container using AWS RIE.
Unifies Zip and Image PackageType through a single warm container pool.
@@ -4715,9 +4719,9 @@ def _execute_function_docker(func: dict, event: dict) -> dict:
"errorType": "Runtime.DockerUnavailable"}, "error": True}
if runtime.startswith(("python", "nodejs")):
logger.warning("docker SDK unavailable - falling back to warm executor")
- return _execute_function_warm(func, event)
+ return _execute_function_warm(func, event, request_id)
logger.warning("docker SDK unavailable - falling back to local subprocess")
- return _execute_function_local(func, event)
+ return _execute_function_local(func, event, request_id)
if _get_docker_client() is None:
if package_type == "Image" or LAMBDA_STRICT:
@@ -4725,8 +4729,8 @@ def _execute_function_docker(func: dict, event: dict) -> dict:
"errorType": "Runtime.DockerError"}, "error": True}
logger.warning("Docker daemon unreachable – falling back")
if runtime.startswith(("python", "nodejs")):
- return _execute_function_warm(func, event)
- return _execute_function_local(func, event)
+ return _execute_function_warm(func, event, request_id)
+ return _execute_function_local(func, event, request_id)
# Zip needs code (Image brings it in the image)
code_zip = func.get("code_zip")
@@ -4787,7 +4791,7 @@ def _execute_function_docker(func: dict, event: dict) -> dict:
time.sleep(0.05)
try:
- result = _invoke_rie(entry["container"], event, timeout)
+ result = _invoke_rie(entry["container"], event, timeout, request_id)
# `timeout` is _invoke_rie's internal signal to this function; pop it so
# only the caller-facing keys survive into the invoke response.
if result.pop("timeout", False):
@@ -4873,12 +4877,13 @@ def _probe_peak_memory_mb(func: dict) -> int:
def _emit_lambda_logs(func: dict, request_id: str, log_text: str,
- error: bool, duration_ms: int) -> None:
+ error: bool, duration_ms: int, *, log_source: str | None = None) -> None:
"""Write handler output to CloudWatch Logs under /aws/lambda/{name}, matching AWS.
- Log group `/aws/lambda/{FunctionName}` is auto-created on first write.
- Stream name follows AWS's format: `{yyyy}/{mm}/{dd}/[{qualifier}]{uuid}`.
- - Each invocation emits START / body / END / REPORT lines like real Lambda.
+ - RIE output already contains platform lines; other executors get a wrapper.
+ - Empty RIE output falls back to the synthetic wrapper so logs remain visible.
Best-effort: a failure to write logs must never break the invocation.
"""
try:
@@ -4916,16 +4921,19 @@ def _emit_lambda_logs(func: dict, request_id: str, log_text: str,
}
stream = group["streams"][stream_name]
- lines: list[str] = [f"START RequestId: {request_id} Version: {qualifier}"]
- if log_text:
- lines.extend(log_text.splitlines())
- lines.append(f"END RequestId: {request_id}")
- peak_mb = _probe_peak_memory_mb(func)
- lines.append(
- f"REPORT RequestId: {request_id}\tDuration: {duration_ms} ms\t"
- f"Billed Duration: {duration_ms} ms\tMemory Size: "
- f"{config.get('MemorySize', 128)} MB\tMax Memory Used: {peak_mb} MB"
- )
+ if log_source == "rie" and log_text:
+ lines = log_text.splitlines()
+ else:
+ lines = [f"START RequestId: {request_id} Version: {qualifier}"]
+ if log_text:
+ lines.extend(log_text.splitlines())
+ lines.append(f"END RequestId: {request_id}")
+ peak_mb = _probe_peak_memory_mb(func)
+ lines.append(
+ f"REPORT RequestId: {request_id}\tDuration: {duration_ms} ms\t"
+ f"Billed Duration: {duration_ms} ms\tMemory Size: "
+ f"{config.get('MemorySize', 128)} MB\tMax Memory Used: {peak_mb} MB"
+ )
for line in lines:
stream["events"].append({"timestamp": now_ms, "message": line, "ingestionTime": now_ms})
if stream["firstEventTimestamp"] is None:
@@ -5040,6 +5048,7 @@ def _execute_function(func: dict, event: dict) -> dict:
# the slot for a single invocation.
return _run_with_function_config_scope(config, _execute_function, func, event)
+ # One invocation identity for the context, Runtime API and platform logs.
request_id = new_uuid()
started = time.time()
@@ -5081,11 +5090,11 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,
if proxy_url:
result = _execute_function_proxy(func, event, proxy_url, request_id)
elif LAMBDA_STRICT:
- result = _execute_function_docker(func, event)
+ result = _execute_function_docker(func, event, request_id)
elif config.get("PackageType") == "Image" and config.get("ImageUri"):
- result = _execute_function_docker(func, event)
+ result = _execute_function_docker(func, event, request_id)
elif LAMBDA_EXECUTOR == "docker":
- result = _execute_function_docker(func, event)
+ result = _execute_function_docker(func, event, request_id)
else:
runtime = config.get("Runtime", "python3.12")
if runtime.startswith("provided"):
@@ -5095,17 +5104,17 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,
# invocations keep the one-shot executor. python and nodejs carry
# that context in the event instead, which is why they can be pooled.
if _durable_ctx.get():
- result = _execute_function_provided(func, event)
+ result = _execute_function_provided(func, event, request_id)
else:
result = _execute_function_provided_warm(func, event, request_id)
elif runtime.startswith(("python", "nodejs")):
# Durable invocations included: their per-call context rides in
# the event, so the pooled worker can serve them.
- result = _execute_function_warm(func, event)
+ result = _execute_function_warm(func, event, request_id)
else:
# java*/dotnet*/ruby* need the real RIE image — there's no
# in-process executor that can run JVM bytecode or .NET IL.
- result = _execute_function_docker(func, event)
+ result = _execute_function_docker(func, event, request_id)
duration_ms = int((time.time() - started) * 1000)
_emit_lambda_logs(
@@ -5113,6 +5122,7 @@ def _execute_function_dispatch(func: dict, config: dict, event: dict,
result.get("log", "") if isinstance(result, dict) else "",
bool(result.get("error")) if isinstance(result, dict) else False,
duration_ms,
+ log_source=result.get("log_source") if isinstance(result, dict) else None,
)
return result
@@ -5238,7 +5248,7 @@ def _execute_function_proxy(func: dict, event: dict, url: str, request_id: str)
return {"body": text}
-def _execute_function_warm(func: dict, event: dict) -> dict:
+def _execute_function_warm(func: dict, event: dict, request_id: str) -> dict:
"""Execute a Lambda function using the warm worker pool (Python + Node.js)."""
config = func.get("config") or func
code_zip = func.get("code_zip")
@@ -5274,7 +5284,7 @@ def _execute_function_warm(func: dict, event: dict) -> dict:
# the values ride the envelope and the bootstrap applies them per call.
result = worker.invoke(
event,
- new_uuid(),
+ request_id,
trace_id=_xray_trace_id_for_invocation(config),
invoked_function_arn=config.get("FunctionArn", ""),
depth=_invoke_depth.get(),
@@ -5348,7 +5358,7 @@ def _provided_worker_env(config: dict, code_dir: str, port: int) -> dict:
def _execute_function_provided_warm(func: dict, event: dict,
- request_id: str | None = None) -> dict:
+ request_id: str) -> dict:
"""Execute a ``provided.*`` Lambda on a pooled, reused environment."""
config = func.get("config") or func
code_zip = func.get("code_zip")
@@ -5368,7 +5378,7 @@ def _execute_function_provided_warm(func: dict, event: dict,
# Invocation metadata belongs in Runtime API headers, not in the user
# payload (which need not be a dict and may contain similarly named keys).
result = worker.invoke(
- event, request_id or new_uuid(),
+ event, request_id,
trace_id=_xray_trace_id_for_invocation(config),
invoked_function_arn=config.get("FunctionArn", ""),
)
@@ -5405,7 +5415,7 @@ def _execute_function_provided_warm(func: dict, event: dict,
release_worker(worker)
-def _execute_function_provided(func: dict, event: dict) -> dict:
+def _execute_function_provided(func: dict, event: dict, request_id: str) -> dict:
"""Execute a provided-runtime Lambda (Go/Rust binary) via a minimal Lambda Runtime API."""
config = func.get("config") or func
code_zip = func.get("code_zip")
@@ -5425,7 +5435,6 @@ def _execute_function_provided(func: dict, event: dict) -> dict:
# Shared state for the Runtime API
result_holder = {"response": None, "error": None}
event_json = json.dumps(event)
- request_id = new_uuid()
event_served = threading.Event()
response_received = threading.Event()
server_ready = threading.Event()
@@ -5565,7 +5574,7 @@ def _execute_function_provided(func: dict, event: dict) -> dict:
return {"body": {"errorMessage": str(e), "errorType": type(e).__name__}, "error": True}
-def _execute_function_local(func: dict, event: dict) -> dict:
+def _execute_function_local(func: dict, event: dict, request_id: str) -> dict:
"""Execute a Lambda function in a one-shot subprocess (fallback for unsupported runtimes)."""
config = func.get("config") or func
code_zip = func.get("code_zip")
@@ -5685,6 +5694,7 @@ def _execute_function_local(func: dict, event: dict) -> dict:
if _xray_trace_id:
env["_X_AMZN_TRACE_ID"] = _xray_trace_id
env[INVOKE_DEPTH_ENV] = str(_invoke_depth.get())
+ env["_LAMBDA_REQUEST_ID"] = request_id
cmd = ["node", wrapper_path] if is_node else ["python3", wrapper_path]
proc = subprocess.run(
diff --git a/tests/test_lambda.py b/tests/test_lambda.py
index df8348da..b8e94fb7 100644
--- a/tests/test_lambda.py
+++ b/tests/test_lambda.py
@@ -4099,7 +4099,7 @@ def test_lambda_provided_runtime_env_has_function_vars():
},
"code_zip": buf.getvalue(),
}
- result = lmod._execute_function_provided(func, {"ping": "pong"})
+ result = lmod._execute_function_provided(func, {"ping": "pong"}, request_id="request-1")
assert not result.get("error"), result
body = result["body"]
assert body["memory"] == "512"
@@ -4150,7 +4150,7 @@ def test_lambda_provided_runtime_parallel_invocations():
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
results = list(ex.map(
- lambda i: lmod._execute_function_provided(func, {"n": i}), range(8)))
+ lambda i: lmod._execute_function_provided(func, {"n": i}, request_id=f"request-{i}"), range(8)))
for i, r in enumerate(results):
assert not r.get("error"), f"invocation {i} failed: {r}"
@@ -5352,7 +5352,7 @@ def test_execute_function_uses_function_config_region_for_logs(monkeypatch):
monkeypatch.setattr(
lsvc,
"_execute_function_warm",
- lambda _func, _event: {"body": {"ok": True}, "log": "ran in target region"},
+ lambda _func, _event, request_id: {"body": {"ok": True}, "log": "ran in target region"},
)
cwl.reset()
try:
@@ -7011,7 +7011,7 @@ def test_warm_invocation_leaves_the_caller_event_untouched(monkeypatch):
event = {"input": "hi"}
token = lsvc._durable_ctx.set({"arn": "exec-arn", "token": "tok", "name": "exec"})
try:
- result = lsvc._execute_function_warm({"config": config, "code_zip": b"zip"}, event)
+ result = lsvc._execute_function_warm({"config": config, "code_zip": b"zip"}, event, request_id="request-1")
finally:
lsvc._durable_ctx.reset(token)
@@ -7123,7 +7123,7 @@ def test_lambda_strict_hard_fails_when_docker_unavailable(monkeypatch):
"Timeout": 3,
"MemorySize": 128,
}, "code_zip": b"\x00"}
- result = lsvc._execute_function_docker(func, {"k": "v"})
+ result = lsvc._execute_function_docker(func, {"k": "v"}, request_id="request-1")
assert result.get("error") is True
assert result["body"]["errorType"] == "Runtime.DockerUnavailable"
@@ -7133,7 +7133,7 @@ def test_lambda_permissive_falls_back_to_warm_without_docker(monkeypatch):
monkeypatch.setattr(lsvc, "LAMBDA_STRICT", False)
monkeypatch.setattr(lsvc, "_docker_available", False)
called = {"warm": False}
- def _fake_warm(func, event):
+ def _fake_warm(func, event, request_id):
called["warm"] = True
return {"body": {"ok": True}}
monkeypatch.setattr(lsvc, "_execute_function_warm", _fake_warm)
@@ -7145,7 +7145,7 @@ def test_lambda_permissive_falls_back_to_warm_without_docker(monkeypatch):
"Timeout": 3,
"MemorySize": 128,
}, "code_zip": b"\x00"}
- lsvc._execute_function_docker(func, {})
+ lsvc._execute_function_docker(func, {}, request_id="request-1")
assert called["warm"] is True
@@ -10584,7 +10584,7 @@ def test_invoke_rie_rewrites_custom_resource_response_url():
"ResourceProperties": {},
}
with patch("urllib.request.urlopen", _fake_urlopen):
- _invoke_rie(_FakeContainer(), event, timeout=5)
+ _invoke_rie(_FakeContainer(), event, timeout=5, request_id="request-1")
assert captured["body"]["ResponseURL"] == \
"http://host.docker.internal:4566/_ministack/cfn-response/tok"
@@ -10666,7 +10666,7 @@ def test_invoke_rie_reports_an_init_error_instead_of_retrying_it():
started = time.time()
with patch("urllib.request.urlopen", _fake_urlopen):
# Timeout=900 is legal on AWS, and used to buy 9020 retries at 0.1s.
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")
elapsed = time.time() - started
assert len(calls) == 1
@@ -10696,7 +10696,7 @@ def test_invoke_rie_still_retries_a_container_that_is_still_starting():
return _FakeResp()
with patch("urllib.request.urlopen", _fake_urlopen):
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=5)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=5, request_id="request-1")
assert len(calls) == 3
assert result["body"] == {"ok": True}
@@ -10714,7 +10714,7 @@ def test_invoke_rie_does_not_re_run_a_handler_that_timed_out():
raise TimeoutError("timed out")
with patch("urllib.request.urlopen", _fake_urlopen):
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")
assert len(calls) == 1
assert result["error"] is True
@@ -10745,7 +10745,7 @@ def test_invoke_rie_reports_a_non_json_502_body():
)
with patch("urllib.request.urlopen", _fake_urlopen):
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")
assert result["error"] is True
assert result["function_error"] == "Unhandled"
@@ -10766,7 +10766,7 @@ def test_invoke_rie_non_json_502_without_an_error_type_header():
)
with patch("urllib.request.urlopen", _fake_urlopen):
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=3, request_id="request-1")
assert result["body"] == {
"errorMessage": "Lambda RIE returned HTTP 502: not json",
@@ -10812,10 +10812,10 @@ def test_lambda_docker_timeout_recycles_the_container(monkeypatch):
try:
with patch("urllib.request.urlopen", _fake_urlopen):
first = lsvc._execute_function_docker(
- {"config": config, "code_zip": b"zip"}, {}
+ {"config": config, "code_zip": b"zip"}, {}, "request-1",
)
second = lsvc._execute_function_docker(
- {"config": config, "code_zip": b"zip"}, {}
+ {"config": config, "code_zip": b"zip"}, {}, "request-1",
)
finally:
lsvc._pool_kill_function("000000000000", name)
@@ -10848,7 +10848,7 @@ def test_invoke_rie_gives_up_on_an_unreachable_container(monkeypatch):
started = time.time()
with patch("urllib.request.urlopen", _fake_urlopen):
- result = lsvc._invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)
+ result = lsvc._invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")
elapsed = time.time() - started
assert elapsed < 5
@@ -13756,7 +13756,7 @@ def test_provided_failure_is_scoped_and_not_retried(monkeypatch, isolated_pool):
unrelated.append(worker)
fallback = Mock()
monkeypatch.setattr(lambda_svc, "_execute_function_provided", fallback)
- result = lambda_svc._execute_function_provided_warm({"config": config, "code_zip": b"zip"}, {})
+ result = lambda_svc._execute_function_provided_warm({"config": config, "code_zip": b"zip"}, {}, request_id="request-1")
assert result["error"] is True
assert result["body"]["errorMessage"] == "bootstrap crashed"
assert fail.call_count == 1
@@ -13864,11 +13864,11 @@ def test_docker_unavailable_fallback_carries_the_durable_context(monkeypatch, un
func = {"config": config, "code_zip": b"zip"}
token = lambda_svc._durable_ctx.set({"arn": "exec-arn", "token": "tok", "name": "exec"})
try:
- assert lambda_svc._execute_function_docker(func, {}) == {"body": "warm"}
+ assert lambda_svc._execute_function_docker(func, {}, request_id="request-1") == {"body": "warm"}
overlay = lambda_svc._durable_env_overlay()
finally:
lambda_svc._durable_ctx.reset(token)
- warm.assert_called_once_with(func, {})
+ warm.assert_called_once_with(func, {}, "request-1")
one_shot.assert_not_called()
assert overlay == {
"AWS_LAMBDA_DURABLE_EXECUTION_ARN": "exec-arn",
@@ -14251,7 +14251,7 @@ def test_invoke_rie_reports_a_bare_string_timeout_as_a_function_error():
return b"Task timed out after 300.00 seconds"
with patch("urllib.request.urlopen", lambda req, timeout=None: _FakeResp()):
- result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900)
+ result = _invoke_rie(_RieFakeContainer(), {"k": "v"}, timeout=900, request_id="request-1")
assert result["error"] is True
assert result["function_error"] == "Unhandled"
@@ -14356,9 +14356,12 @@ def test_invocation_retains_alias_without_mutating_version(monkeypatch, asynchro
original = deepcopy(function)
monkeypatch.setitem(svc._functions, name, function)
monkeypatch.setattr(svc, "_emit_lambda_metrics", lambda *args, **kwargs: None)
- monkeypatch.setattr(svc, "_execute_function_with_config_scope", svc._execute_function_warm)
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "local")
+ monkeypatch.setattr(svc, "LAMBDA_STRICT", False)
+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: None)
+ monkeypatch.setattr(svc, "_emit_lambda_logs", lambda *args, **kwargs: None)
captured = []
- monkeypatch.setattr(svc, "invoke_async_with_retry", lambda f, e: captured.append(svc._execute_function_warm(f, e)))
+ monkeypatch.setattr(svc, "invoke_async_with_retry", lambda f, e: captured.append(svc._execute_function_with_config_scope(f, e)))
try:
# Alternate aliases sharing one published version, including warm reuse.
for qualifier in ["worker", "other", "1", "worker", None, "$LATEST"]:
diff --git a/tests/test_lambda_invocation_logs.py b/tests/test_lambda_invocation_logs.py
new file mode 100644
index 00000000..2a7def91
--- /dev/null
+++ b/tests/test_lambda_invocation_logs.py
@@ -0,0 +1,322 @@
+"""Invocation IDs and platform logs: AWS docs and local RIE observations.
+
+Docker tests use real RIE containers; no live AWS account validation is claimed.
+"""
+import asyncio
+import base64
+import io
+import json
+import re
+import shutil
+import time
+import urllib.error
+import uuid
+import zipfile
+from unittest.mock import Mock
+
+import pytest
+
+from ministack.core import lambda_runtime
+from ministack.core.responses import AccountRegionScopedDict
+from ministack.services import cloudwatch_logs as cwl
+from ministack.services import lambda_svc as svc
+
+
+def log_state(monkeypatch):
+ monkeypatch.setattr(cwl, "_log_groups", AccountRegionScopedDict())
+ monkeypatch.setattr(cwl, "_fanout_to_subscription_filters", Mock())
+ monkeypatch.setattr(svc, "_probe_peak_memory_mb", lambda func: 12)
+ monkeypatch.setattr(svc, "_ensure_reaper_thread", lambda: None)
+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: None)
+ monkeypatch.setattr(svc, "LAMBDA_STRICT", False)
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "local")
+
+
+def _function(runtime="python3.12", code_zip=None):
+ name = "invocation-logs-" + uuid.uuid4().hex
+ return {"config": {
+ "FunctionName": name,
+ "FunctionArn": svc._func_arn(name),
+ "Runtime": runtime, "Handler": "index.handler", "Version": "$LATEST",
+ "MemorySize": 128, "Timeout": 10, "PackageType": "Zip",
+ }, "code_zip": code_zip}
+
+
+def _messages(func):
+ group = cwl._log_groups["/aws/lambda/" + func["config"]["FunctionName"]]
+ return [e["message"] for s in group["streams"].values() for e in s["events"]]
+
+
+def _assert_platform_lines(messages, request_id):
+ for kind in ("START", "END", "REPORT"):
+ ids = [m.group(1) for line in messages
+ if (m := re.match(rf"^{kind} RequestId: (\S+)", line))]
+ assert ids == [request_id]
+
+
[email protected]("mode,target", [
+ ("python", "_execute_function_warm"), ("node", "_execute_function_warm"),
+ ("docker", "_execute_function_docker"), ("strict", "_execute_function_docker"),
+ ("image", "_execute_function_docker"), ("other", "_execute_function_docker"),
+ ("provided", "_execute_function_provided_warm"),
+ ("durable", "_execute_function_provided"), ("proxy", "_execute_function_proxy"),
+])
+def test_execute_function_generates_one_request_id(monkeypatch, log_state, mode, target):
+ func = _function()
+ if mode == "node":
+ func["config"]["Runtime"] = "nodejs22.x"
+ elif mode == "other":
+ func["config"]["Runtime"] = "ruby3.3"
+ elif mode in ("provided", "durable"):
+ func["config"]["Runtime"] = "provided.al2023"
+ elif mode == "image":
+ func["config"].update(PackageType="Image", ImageUri="example:latest")
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker" if mode == "docker" else "local")
+ monkeypatch.setattr(svc, "LAMBDA_STRICT", mode == "strict")
+ if mode == "proxy":
+ monkeypatch.setattr(svc, "_proxy_url_for", lambda config: "http://proxy")
+ generated = Mock(return_value="invocation-id")
+ executor = Mock(return_value={"body": {"ok": True}, "log": "user output"})
+ emit = Mock()
+ monkeypatch.setattr(svc, "new_uuid", generated)
+ monkeypatch.setattr(svc, target, executor)
+ monkeypatch.setattr(svc, "_emit_lambda_logs", emit)
+ token = svc._durable_ctx.set({"arn": "execution"} if mode == "durable" else None)
+ try:
+ svc._execute_function(func, {"input": 1})
+ finally:
+ svc._durable_ctx.reset(token)
+ generated.assert_called_once_with()
+ args = (func, {"input": 1}, "http://proxy", "invocation-id") if mode == "proxy" else (
+ func, {"input": 1}, "invocation-id")
+ executor.assert_called_once_with(*args)
+ assert emit.call_args.args[1] == "invocation-id"
+
+
[email protected]("unavailable", ["sdk", "daemon"])
[email protected]("runtime,target", [
+ ("python3.12", "_execute_function_warm"),
+ ("nodejs22.x", "_execute_function_warm"),
+ ("ruby3.3", "_execute_function_local"),
+])
+def test_docker_fallback_preserves_id_and_synthetic_logs(
+ monkeypatch, log_state, unavailable, runtime, target):
+ func = _function(runtime, b"zip")
+ monkeypatch.setattr(svc, "_docker_available", unavailable != "sdk")
+ monkeypatch.setattr(svc, "_get_docker_client", lambda: None)
+ executor = Mock(return_value={"body": {}, "log": "USER_PRINT fallback"})
+ monkeypatch.setattr(svc, target, executor)
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")
+ svc._execute_function_dispatch(func, func["config"], {}, "same-id", time.time())
+ executor.assert_called_once_with(func, {}, "same-id")
+ _assert_platform_lines(_messages(func), "same-id")
+
+
[email protected]("log_source,error", [(None, False), ("rie", False), ("rie", True)])
+def test_invoke_preserves_log_provenance_and_tail(monkeypatch, log_state, log_source, error):
+ func = _function()
+ raw = "START RequestId: same-id Version: $LATEST\nUSER_PRINT once\nEND RequestId: same-id\nREPORT RequestId: same-id\tDuration: 1.25 ms"
+ monkeypatch.setitem(svc._functions, func["config"]["FunctionName"], func)
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")
+ monkeypatch.setattr(svc, "_execute_function_docker", Mock(return_value={
+ "body": {}, "log": raw, "log_source": log_source, "error": error}))
+ monkeypatch.setattr(svc, "_emit_lambda_metrics", lambda *args, **kwargs: None)
+ monkeypatch.setattr(svc, "new_uuid", lambda: "same-id")
+ status, headers, _ = asyncio.run(svc._invoke(func["config"]["FunctionName"], {}, {"x-amz-log-type": "Tail"}))
+ assert status == 200
+ assert ("X-Amz-Function-Error" in headers) == error
+ tail = base64.b64decode(headers["X-Amz-Log-Result"]).decode().splitlines()
+ assert tail == raw.splitlines()
+ messages = _messages(func)
+ # User-written platform-looking lines remain user logs without RIE metadata.
+ assert (messages if log_source == "rie" else messages[1:-2]) == tail
+ if log_source == "rie":
+ _assert_platform_lines(messages, "same-id")
+ else:
+ _assert_platform_lines([messages[0], *messages[-2:]], "same-id")
+ assert len(messages) == len(tail) + 3
+ forwarded = cwl._fanout_to_subscription_filters.call_args.args[2]
+ assert [e["message"] for e in forwarded] == messages
+
+
[email protected]("raw", ["", "partial runtime output"])
+def test_rie_empty_and_partial_logs_are_handled_without_text_parsing(log_state, raw):
+ func = _function()
+ svc._emit_lambda_logs(func, "same-id", raw, True, 1, log_source="rie")
+ if raw:
+ assert _messages(func) == [raw]
+ else:
+ _assert_platform_lines(_messages(func), "same-id")
+
+
[email protected]("outcome", ["ok", "handler-error", "init-error", "read-timeout", "not-running"])
+def test_invoke_rie_passes_request_id_and_marks_actual_rie_output(monkeypatch, outcome):
+ container = Mock()
+ container.status = "exited" if outcome == "not-running" else "running"
+ container.attrs = {"NetworkSettings": {"Networks": {}}}
+ container.ports = {"8080/tcp": [{"HostPort": "12345"}]}
+ container.logs.return_value = b"opaque output"
+ monkeypatch.setattr(svc, "_running_in_container", lambda: False)
+ monkeypatch.setattr(svc, "LAMBDA_DOCKER_NETWORK", "")
+ response = Mock(headers={})
+ response.read.return_value = json.dumps(
+ {"errorType": "ValueError", "errorMessage": "failed"} if outcome == "handler-error" else {"ok": True}
+ ).encode()
+ seen = []
+
+ def post(req, timeout):
+ seen.append(req)
+ if outcome == "init-error":
+ raise urllib.error.HTTPError(req.full_url, 502, "Bad Gateway", {}, io.BytesIO(b"{}"))
+ if outcome == "read-timeout":
+ raise TimeoutError("timed out")
+ return response
+
+ monkeypatch.setattr(svc.urllib.request, "urlopen", post)
+ result = svc._invoke_rie(container, {"input": 1}, 3, "same-id")
+ assert result["log"] == "opaque output"
+ if outcome == "not-running":
+ assert not seen
+ assert "log_source" not in result
+ else:
+ assert len(seen) == 1
+ assert dict(seen[0].header_items())["X-amzn-requestid"] == "same-id"
+ assert json.loads(seen[0].data) == {"input": 1}
+ assert result["log_source"] == "rie"
+
+
+_PYTHON_HANDLER = """import logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+def handler(event, context):
+ print('USER_PRINT ' + event['marker'], flush=True)
+ logger.info('USER_INFO %s', event['marker'])
+ return {'request_id': context.aws_request_id, 'stream': context.log_stream_name}
+"""
+_NODE_HANDLER = """exports.handler = async (event, context) => {
+ return {request_id: context.awsRequestId, stream: context.logStreamName};
+};
+"""
+_PROVIDED_BOOTSTRAP = """#!/usr/bin/env python3
+import json, os, urllib.request
+base = 'http://' + os.environ['AWS_LAMBDA_RUNTIME_API'] + '/2018-06-01/runtime/'
+while True:
+ response = urllib.request.urlopen(base + 'invocation/next')
+ request_id = response.headers['Lambda-Runtime-Aws-Request-Id']
+ response.read()
+ payload = json.dumps({'request_id': request_id}).encode()
+ urllib.request.urlopen(urllib.request.Request(base + 'invocation/' + request_id + '/response', data=payload)).read()
+ if os.environ.get('PROBE_ONE_SHOT'):
+ break
+"""
+
+
+def _code_zip():
+ output = io.BytesIO()
+ with zipfile.ZipFile(output, "w") as archive:
+ archive.writestr("index.py", _PYTHON_HANDLER)
+ archive.writestr("index.js", _NODE_HANDLER)
+ bootstrap = zipfile.ZipInfo("bootstrap")
+ bootstrap.external_attr = 0o100755 << 16
+ archive.writestr(bootstrap, _PROVIDED_BOOTSTRAP)
+ return output.getvalue()
+
+
+def _assert_two_invocations(func, *, rie=False):
+ previous_id = None
+ for seq in range(2):
+ marker = str(seq)
+ before = len(_messages(func)) if seq else 0
+ result = svc._execute_function(func, {"marker": marker})
+ assert not result.get("error"), result
+ request_id = result["body"]["request_id"]
+ assert request_id != previous_id
+ if func["config"]["Runtime"].startswith(("python", "nodejs")):
+ assert request_id != result["body"]["stream"]
+ messages = _messages(func)[before:]
+ _assert_platform_lines(messages, request_id)
+ if rie:
+ assert result["log_source"] == "rie"
+ assert messages == result["log"].splitlines()
+ if func["config"]["Runtime"].startswith("python"):
+ for kind in ("USER_PRINT", "USER_INFO"):
+ assert sum(kind + " " + marker in line for line in messages) == 1
+ previous_id = request_id
+
+
[email protected]("runtime", ["python3.12", "nodejs22.x"])
[email protected]("executor", ["warm", "local"])
+def test_real_python_and_node_invocations_share_platform_request_id(monkeypatch, log_state, runtime, executor):
+ if runtime.startswith("nodejs") and not shutil.which("node"):
+ pytest.skip("Node.js is required")
+ if executor == "local" and runtime.startswith("python") and not shutil.which("python3"):
+ pytest.skip("local executor requires python3")
+ if executor == "local":
+ monkeypatch.setattr(svc, "_execute_function_warm", svc._execute_function_local)
+ func = _function(runtime, _code_zip())
+ try:
+ _assert_two_invocations(func)
+ finally:
+ lambda_runtime.invalidate_worker(func["config"]["FunctionName"])
+
+
[email protected]("one_shot", [False, True])
+def test_real_provided_runtime_receives_platform_request_id(monkeypatch, log_state, one_shot):
+ if not shutil.which("python3"):
+ pytest.skip("provided bootstrap requires /usr/bin/env python3")
+ func = _function("provided.al2023", _code_zip())
+ if one_shot:
+ func["config"]["Environment"] = {"Variables": {"PROBE_ONE_SHOT": "1"}}
+ token = svc._durable_ctx.set({"arn": "execution"} if one_shot else None)
+ try:
+ _assert_two_invocations(func)
+ finally:
+ svc._durable_ctx.reset(token)
+ lambda_runtime.invalidate_worker(func["config"]["FunctionName"])
+
+
[email protected](scope="module")
+def python_lambda_image(tmp_path_factory):
+ docker = pytest.importorskip("docker")
+ try:
+ client = docker.from_env()
+ client.ping()
+ except docker.errors.DockerException as exc:
+ pytest.skip(f"Docker daemon unavailable: {exc}")
+ base = "public.ecr.aws/lambda/python:3.12"
+ try:
+ client.images.get(base)
+ except docker.errors.ImageNotFound:
+ client.images.pull(base)
+ tag = "ministack-log-test:" + uuid.uuid4().hex
+ build_dir = tmp_path_factory.mktemp("lambda-image")
+ (build_dir / "Dockerfile").write_text(
+ 'FROM ' + base + '\nCOPY index.py /var/task/index.py\nCMD ["index.handler"]\n', encoding="utf-8")
+ (build_dir / "index.py").write_text(_PYTHON_HANDLER, encoding="utf-8")
+ client.images.build(path=str(build_dir), tag=tag, rm=True)
+ try:
+ yield tag
+ finally:
+ client.images.remove(tag)
+ client.close()
+
+
[email protected]("package_type", ["Zip", "Image"])
+def test_real_docker_invocations_have_one_platform_log_sequence(monkeypatch, log_state, request, package_type):
+ if svc._get_docker_client() is None:
+ pytest.skip("Docker daemon unavailable")
+ monkeypatch.setattr(svc, "LAMBDA_EXECUTOR", "docker")
+ monkeypatch.setattr(svc, "_KEEPALIVE_KILL_ON_RELEASE", False)
+ spawn = Mock(wraps=svc._spawn_lambda_container)
+ monkeypatch.setattr(svc, "_spawn_lambda_container", spawn)
+ func = _function(code_zip=_code_zip())
+ if package_type == "Image":
+ func["config"].update(PackageType="Image", ImageUri=request.getfixturevalue("python_lambda_image"))
+ try:
+ _assert_two_invocations(func, rie=True)
+ spawn.assert_called_once() # The second invocation must reuse the environment.
+ finally:
+ svc._pool_kill_function("000000000000", func["config"]["FunctionName"])