Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
perf(lambda): store code_zip bytes as content-addressed blob files (#947)
W. H. Wangecc07f8e831c
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.pyindex d243f524..3adb034c 100644--- a/ministack/services/lambda_svc.py+++ b/ministack/services/lambda_svc.py@@ -48,7 +48,7 @@ from typing import Anyfrom urllib.parse import unquotefrom ministack.core.lambda_runtime import get_or_create_worker, invalidate_worker-from ministack.core.persistence import PERSIST_STATE, load_state+from ministack.core.persistence import PERSIST_STATE, STATE_DIR, load_statefrom ministack.core.responses import (_12_DIGIT_RE,AccountScopedDict,@@ -223,19 +223,136 @@ _poller_lock = threading.Lock()# ── Persistence ────────────────────────────────────────────+# Lambda code zips are stored on disk as content-addressed blob files under+# ${STATE_DIR}/lambda-blobs/{sha256}.zip; ``lambda.json`` only carries the+# sha256 reference. Inline base64 in the state file (the previous shape)+# scales poorly: a single ~30 MB zip multiplied by N functions encodes ~1.3×+# larger as base64 and forces a single-shot JSON parse on every restart,+# which pushes memory to ~2× the encoded size during decode.+#+# Content-addressing means two functions with identical bytes share one+# blob file. The orphan sweep at the end of ``get_state()`` removes blob+# files no longer referenced by any function or version (e.g. previous+# ``UpdateFunctionCode`` generations).+#+# Backward compatibility: ``restore_state`` accepts the legacy inline+# base64 shape (``"code_zip": "<b64>"``) so an upgrade in place works+# without a one-shot migration step.++CODE_BLOB_DIR = os.path.join(STATE_DIR, "lambda-blobs")+_BLOB_REF_KEY = "code_blob_ref"+++def _write_code_blob(code_zip: bytes) -> str:+ """Write ``code_zip`` to a content-addressed file. Returns the sha256+ hex digest used as the file name. Idempotent: an existing file with+ matching content (same sha) is left in place."""+ sha = hashlib.sha256(code_zip).hexdigest()+ os.makedirs(CODE_BLOB_DIR, exist_ok=True)+ path = os.path.join(CODE_BLOB_DIR, f"{sha}.zip")+ if os.path.exists(path):+ return sha+ tmp = path + ".tmp"+ try:+ with open(tmp, "wb") as f:+ f.write(code_zip)+ os.replace(tmp, path)+ except BaseException:+ # Avoid leaving partial writes that would later be picked up as if+ # they were valid blobs.+ try:+ os.remove(tmp)+ except OSError:+ pass+ raise+ return sha+++def _read_code_blob(sha: str) -> bytes | None:+ """Read a blob by sha256. Returns ``None`` and logs an error when the+ file is missing or unreadable — restore should degrade the affected+ function rather than abort the whole load."""+ path = os.path.join(CODE_BLOB_DIR, f"{sha}.zip")+ try:+ with open(path, "rb") as f:+ return f.read()+ except FileNotFoundError:+ logger.error("Lambda persistence: code blob %s missing at %s", sha, path)+ return None+ except OSError as e:+ logger.error("Lambda persistence: failed to read code blob %s: %s", sha, e)+ return None+++def _prune_orphan_code_blobs(referenced: set[str]) -> None:+ """Remove blob files whose sha is not in the ``referenced`` set. Called+ at the tail of ``get_state()`` so the on-disk set always reflects the+ current ``_functions`` graph. Logs each removal at debug level; failures+ are non-fatal (best-effort hygiene)."""+ if not os.path.isdir(CODE_BLOB_DIR):+ return+ for entry in os.listdir(CODE_BLOB_DIR):+ if not entry.endswith(".zip"):+ continue+ sha = entry[: -len(".zip")]+ if sha in referenced:+ continue+ try:+ os.remove(os.path.join(CODE_BLOB_DIR, entry))+ logger.debug("Lambda persistence: pruned orphan blob %s", sha)+ except OSError as e:+ logger.warning("Lambda persistence: failed to prune blob %s: %s", sha, e)+++def _externalize_code_zip(func: dict, referenced: set[str]) -> None:+ """Replace ``func['code_zip']`` (bytes) with a blob ref in place, and+ record the sha in ``referenced`` so the orphan sweep keeps the file."""+ code = func.get("code_zip")+ if not code:+ return+ if isinstance(code, bytes):+ sha = _write_code_blob(code)+ func["code_zip"] = {_BLOB_REF_KEY: sha}+ referenced.add(sha)+ elif isinstance(code, dict) and _BLOB_REF_KEY in code:+ # Already a ref (e.g. ``get_state`` called twice without a restore+ # in between). Keep the file referenced.+ referenced.add(code[_BLOB_REF_KEY])+ # Legacy inline base64 strings are not re-serialized here — they reach+ # ``get_state`` only if a caller copied them directly into _functions+ # without going through CreateFunction (no real code path does this).+++def _internalize_code_zip(func: dict) -> None:+ """Replace a blob ref or legacy base64 string with bytes, in place."""+ code = func.get("code_zip")+ if not code:+ return+ if isinstance(code, dict) and _BLOB_REF_KEY in code:+ func["code_zip"] = _read_code_blob(code[_BLOB_REF_KEY])+ elif isinstance(code, str):+ # Legacy persistence shape (inline base64).+ func["code_zip"] = base64.b64decode(code)++def get_state():- """Return JSON-serializable state. code_zip bytes are base64-encoded."""+ """Return JSON-serializable Lambda state.++ Function code (``code_zip``) is written to disk as content-addressed+ blobs under ``${STATE_DIR}/lambda-blobs/`` and replaced in the returned+ state with ``{"code_blob_ref": "<sha256>"}`` markers. Unreferenced blobs+ from prior generations are pruned."""from ministack.core.responses import AccountScopedDictfuncs = AccountScopedDict()+ referenced: set[str] = set()# Iterate _data directly to capture ALL accounts, not just current request contextfor scoped_key, func in _functions._data.items():f = copy.deepcopy(func)- if f.get("code_zip") and isinstance(f["code_zip"], bytes):- f["code_zip"] = base64.b64encode(f["code_zip"]).decode()+ _externalize_code_zip(f, referenced)for ver in f.get("versions", {}).values():- if ver.get("code_zip") and isinstance(ver["code_zip"], bytes):- ver["code_zip"] = base64.b64encode(ver["code_zip"]).decode()+ _externalize_code_zip(ver, referenced)funcs._data[scoped_key] = f+ _prune_orphan_code_blobs(referenced)return {"functions": funcs,"layers": copy.deepcopy(_layers),@@ -256,19 +373,15 @@ def restore_state(data):funcs = data.get("functions", {})if isinstance(funcs, AccountScopedDict):for scoped_key, func in funcs._data.items():- if func.get("code_zip") and isinstance(func["code_zip"], str):- func["code_zip"] = base64.b64decode(func["code_zip"])+ _internalize_code_zip(func)for ver in func.get("versions", {}).values():- if ver.get("code_zip") and isinstance(ver["code_zip"], str):- ver["code_zip"] = base64.b64decode(ver["code_zip"])+ _internalize_code_zip(ver)_functions._data[scoped_key] = funcelse:for name, func in funcs.items():- if func.get("code_zip") and isinstance(func["code_zip"], str):- func["code_zip"] = base64.b64decode(func["code_zip"])+ _internalize_code_zip(func)for ver in func.get("versions", {}).values():- if ver.get("code_zip") and isinstance(ver["code_zip"], str):- ver["code_zip"] = base64.b64decode(ver["code_zip"])+ _internalize_code_zip(ver)_functions[name] = func_layers.update(data.get("layers", {}))_esms.update(data.get("esms", {}))diff --git a/tests/test_lambda.py b/tests/test_lambda.pyindex cecdbfbb..fdb40149 100644--- a/tests/test_lambda.py+++ b/tests/test_lambda.py@@ -4510,6 +4510,194 @@ def test_lambda_durable_persistence_round_trip():lambda_durable.restore_state(original)+# ---------------------------------------------------------------------------+# Lambda code_zip persistence — content-addressed blob storage.+# code_zip bytes are written to ${STATE_DIR}/lambda-blobs/{sha256}.zip; the+# returned state holds only the sha reference. The in-memory _functions+# shape is unchanged (still bytes after restore) so invoke / update / delete+# paths see no difference.+# ---------------------------------------------------------------------------+++def _make_lambda_record(name: str, code_zip: bytes, versions: dict | None = None) -> dict:+ """Build a _functions entry matching what CreateFunction stores."""+ return {+ "config": {+ "FunctionName": name,+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{name}",+ "Runtime": "python3.12",+ "Handler": "index.handler",+ },+ "code_zip": code_zip,+ "versions": versions or {},+ "next_version": 1,+ "tags": {},+ "policy": {"Version": "2012-10-17", "Id": "default", "Statement": []},+ }+++def lambda_svc_isolated(tmp_path, monkeypatch):+ """Snapshot lambda_svc state and redirect its blob storage to tmp_path.+ Restores the original state at teardown so tests don't pollute the+ shared in-process module that serves the running ministack."""+ from ministack.services import lambda_svc++ monkeypatch.setattr(lambda_svc, "CODE_BLOB_DIR", str(tmp_path / "lambda-blobs"))+ original = lambda_svc.get_state()+ try:+ lambda_svc._functions._data.clear()+ yield lambda_svc, tmp_path / "lambda-blobs"+ finally:+ lambda_svc._functions._data.clear()+ lambda_svc.restore_state(original)+++def test_lambda_code_zip_round_trip_through_blob_storage(lambda_svc_isolated):+ """Bytes survive an exact get_state → clear → restore_state round-trip."""+ svc, _ = lambda_svc_isolated+ code = b"\x89PNG\r\n\x1a\n" + b"\x00" * 1000 # non-UTF8 bytes+ svc._functions["fn"] = _make_lambda_record("fn", code)++ state = svc.get_state()+ svc._functions._data.clear()+ svc.restore_state(state)++ restored = svc._functions._data[("000000000000", "fn")]+ assert restored["code_zip"] == code+ assert isinstance(restored["code_zip"], bytes)+++def test_lambda_get_state_replaces_code_zip_with_blob_ref(lambda_svc_isolated):+ """The returned state must not inline bytes or base64 — only a sha ref."""+ import hashlib+ svc, blob_dir = lambda_svc_isolated+ code = b"def handler(event, ctx): return 'ok'\n" * 256+ svc._functions["fn"] = _make_lambda_record("fn", code)++ state = svc.get_state()+ fn_state = state["functions"]._data[("000000000000", "fn")]++ assert fn_state["code_zip"] == {"code_blob_ref": hashlib.sha256(code).hexdigest()}+ blob_path = blob_dir / f"{hashlib.sha256(code).hexdigest()}.zip"+ assert blob_path.exists()+ assert blob_path.read_bytes() == code+++def test_lambda_per_version_code_zip_also_externalized(lambda_svc_isolated):+ """PublishVersion-created versions also store code_zip externally."""+ import hashlib+ svc, _ = lambda_svc_isolated+ v1, v2 = b"v1 body", b"v2 body"+ svc._functions["fn"] = _make_lambda_record(+ "fn", v2, versions={"1": {"code_zip": v1, "config": {"Version": "1"}}}+ )++ state = svc.get_state()+ fn_state = state["functions"]._data[("000000000000", "fn")]+ assert fn_state["versions"]["1"]["code_zip"] == {+ "code_blob_ref": hashlib.sha256(v1).hexdigest()+ }++ svc._functions._data.clear()+ svc.restore_state(state)+ restored = svc._functions._data[("000000000000", "fn")]+ assert restored["code_zip"] == v2+ assert restored["versions"]["1"]["code_zip"] == v1+++def test_lambda_identical_code_dedups_to_single_blob(lambda_svc_isolated):+ """Content-addressing means two functions with identical bytes share one+ file. Important when the deps-bundled-into-every-zip pattern produces+ many functions with byte-identical layer payloads."""+ svc, blob_dir = lambda_svc_isolated+ code = b"shared body across two functions"+ svc._functions["fn-a"] = _make_lambda_record("fn-a", code)+ svc._functions["fn-b"] = _make_lambda_record("fn-b", code)++ svc.get_state()++ files = sorted(blob_dir.iterdir())+ assert len(files) == 1, [f.name for f in files]+++def test_lambda_legacy_inline_base64_persistence_still_loads(lambda_svc_isolated):+ """Pre-existing lambda.json files (written before content-addressed+ storage) stored code_zip as inline base64. restore_state must accept+ that shape so an in-place upgrade requires no migration step."""+ import base64 as _b64+ from ministack.core.responses import AccountScopedDict+ svc, _ = lambda_svc_isolated++ code = b"legacy persisted bytes"+ legacy = {+ "functions": AccountScopedDict(),+ "layers": AccountScopedDict(),+ "esms": AccountScopedDict(),+ "function_urls": AccountScopedDict(),+ "kinesis_positions": {},+ "dynamodb_stream_positions": {},+ }+ legacy["functions"]._data[("000000000000", "old-fn")] = {+ "config": {"FunctionName": "old-fn"},+ "code_zip": _b64.b64encode(code).decode(),+ "versions": {},+ }++ svc.restore_state(legacy)++ restored = svc._functions._data[("000000000000", "old-fn")]+ assert restored["code_zip"] == code+++def test_lambda_missing_blob_degrades_without_aborting_restore(lambda_svc_isolated):+ """If a blob file is missing (corrupted volume / partial mount), restore+ must downgrade the affected function (code_zip=None) and continue,+ rather than raise and prevent the whole server from starting."""+ from ministack.core.responses import AccountScopedDict+ svc, _ = lambda_svc_isolated++ state = {+ "functions": AccountScopedDict(),+ "layers": AccountScopedDict(),+ "esms": AccountScopedDict(),+ "function_urls": AccountScopedDict(),+ "kinesis_positions": {},+ "dynamodb_stream_positions": {},+ }+ state["functions"]._data[("000000000000", "orphan")] = {+ "config": {"FunctionName": "orphan"},+ "code_zip": {"code_blob_ref": "0" * 64}, # sha pointing at nothing+ "versions": {},+ }++ svc.restore_state(state)++ assert svc._functions._data[("000000000000", "orphan")]["code_zip"] is None+++def test_lambda_get_state_prunes_orphan_blobs(lambda_svc_isolated):+ """When a function's code is updated, the previous generation's blob+ becomes orphan. The next get_state sweeps it. Without this, repeated+ UpdateFunctionCode (e.g. iterative sandbox redeploys) leaks blob files+ across restarts."""+ import hashlib+ svc, blob_dir = lambda_svc_isolated++ old_code = b"old code"+ new_code = b"new code"+ svc._functions["fn"] = _make_lambda_record("fn", old_code)+ svc.get_state() # writes blob(old_code)+ assert (blob_dir / f"{hashlib.sha256(old_code).hexdigest()}.zip").exists()++ # Simulate UpdateFunctionCode.+ svc._functions["fn"]["code_zip"] = new_code+ svc.get_state() # writes blob(new_code), should remove blob(old_code)++ assert (blob_dir / f"{hashlib.sha256(new_code).hexdigest()}.zip").exists()+ assert not (blob_dir / f"{hashlib.sha256(old_code).hexdigest()}.zip").exists()++def test_lambda_durable_event_wrapped_with_sdk_fields(lam):"""A durable invocation's event payload is wrapped with the fields theaws-durable-execution-sdk-python SDK reads from the Lambda event:diff --git a/tests/test_ministack.py b/tests/test_ministack.pyindex e950158d..85c30dc2 100644--- a/tests/test_ministack.py+++ b/tests/test_ministack.py@@ -364,23 +364,35 @@ def test_ministack_persist_s3_logging_accelerate_request_payment_roundtrip():_s3._bucket_request_payment_config.pop("persist-log-bkt")def test_ministack_persist_lambda_roundtrip():+ import hashlib+ import osfrom ministack.services import lambda_svc as _lam+ code = b"fake-zip-bytes"+ sha = hashlib.sha256(code).hexdigest()_lam._functions["persist-fn"] = {"config": {"FunctionName": "persist-fn", "Runtime": "python3.11"},- "code_zip": b"fake-zip-bytes",+ "code_zip": code,"versions": {},"next_version": 1,}- state = _lam.get_state()- assert "functions" in state- # code_zip should be base64-encoded in state- assert isinstance(state["functions"]["persist-fn"]["code_zip"], str)- _lam._functions.pop("persist-fn")- _lam.restore_state(state)- assert "persist-fn" in _lam._functions- # code_zip should be decoded back to bytes- assert _lam._functions["persist-fn"]["code_zip"] == b"fake-zip-bytes"- _lam._functions.pop("persist-fn")+ try:+ state = _lam.get_state()+ assert "functions" in state+ # code_zip is replaced with a content-addressed blob reference in state;+ # the bytes themselves live under ${STATE_DIR}/lambda-blobs/{sha}.zip.+ assert state["functions"]["persist-fn"]["code_zip"] == {"code_blob_ref": sha}+ _lam._functions.pop("persist-fn")+ _lam.restore_state(state)+ assert "persist-fn" in _lam._functions+ # code_zip is restored back to bytes.+ assert _lam._functions["persist-fn"]["code_zip"] == code+ finally:+ _lam._functions.pop("persist-fn", None)+ # Clean up the blob file written as a side effect of get_state.+ try:+ os.remove(os.path.join(_lam.CODE_BLOB_DIR, f"{sha}.zip"))+ except OSError:+ passdef test_ministack_persist_rds_roundtrip():from ministack.services import rds as _rds