Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

perf(lambda): store code_zip bytes as content-addressed blob files (#947)

W. H. Wangecc07f8e831c
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index d243f524..3adb034c 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -48,7 +48,7 @@ from typing import Any
from urllib.parse import unquote
from ministack.core.lambda_runtime import get_or_create_worker, invalidate_worker
-from ministack.core.persistence import PERSIST_STATE, load_state
+from ministack.core.persistence import PERSIST_STATE, STATE_DIR, load_state
from ministack.core.responses import (
_12_DIGIT_RE,
AccountScopedDict,
@@ -223,19 +223,136 @@ _poller_lock = threading.Lock()
# ── Persistence ────────────────────────────────────────────
+# Lambda code zips are stored on disk as content-addressed blob files under
+# ${STATE_DIR}/lambda-blobs/{sha256}.zip; ``lambda.json`` only carries the
+# sha256 reference. Inline base64 in the state file (the previous shape)
+# scales poorly: a single ~30 MB zip multiplied by N functions encodes ~1.3×
+# larger as base64 and forces a single-shot JSON parse on every restart,
+# which pushes memory to ~2× the encoded size during decode.
+#
+# Content-addressing means two functions with identical bytes share one
+# blob file. The orphan sweep at the end of ``get_state()`` removes blob
+# files no longer referenced by any function or version (e.g. previous
+# ``UpdateFunctionCode`` generations).
+#
+# Backward compatibility: ``restore_state`` accepts the legacy inline
+# base64 shape (``"code_zip": "<b64>"``) so an upgrade in place works
+# without a one-shot migration step.
+
+CODE_BLOB_DIR = os.path.join(STATE_DIR, "lambda-blobs")
+_BLOB_REF_KEY = "code_blob_ref"
+
+
+def _write_code_blob(code_zip: bytes) -> str:
+ """Write ``code_zip`` to a content-addressed file. Returns the sha256
+ hex digest used as the file name. Idempotent: an existing file with
+ matching content (same sha) is left in place."""
+ sha = hashlib.sha256(code_zip).hexdigest()
+ os.makedirs(CODE_BLOB_DIR, exist_ok=True)
+ path = os.path.join(CODE_BLOB_DIR, f"{sha}.zip")
+ if os.path.exists(path):
+ return sha
+ tmp = path + ".tmp"
+ try:
+ with open(tmp, "wb") as f:
+ f.write(code_zip)
+ os.replace(tmp, path)
+ except BaseException:
+ # Avoid leaving partial writes that would later be picked up as if
+ # they were valid blobs.
+ try:
+ os.remove(tmp)
+ except OSError:
+ pass
+ raise
+ return sha
+
+
+def _read_code_blob(sha: str) -> bytes | None:
+ """Read a blob by sha256. Returns ``None`` and logs an error when the
+ file is missing or unreadable — restore should degrade the affected
+ function rather than abort the whole load."""
+ path = os.path.join(CODE_BLOB_DIR, f"{sha}.zip")
+ try:
+ with open(path, "rb") as f:
+ return f.read()
+ except FileNotFoundError:
+ logger.error("Lambda persistence: code blob %s missing at %s", sha, path)
+ return None
+ except OSError as e:
+ logger.error("Lambda persistence: failed to read code blob %s: %s", sha, e)
+ return None
+
+
+def _prune_orphan_code_blobs(referenced: set[str]) -> None:
+ """Remove blob files whose sha is not in the ``referenced`` set. Called
+ at the tail of ``get_state()`` so the on-disk set always reflects the
+ current ``_functions`` graph. Logs each removal at debug level; failures
+ are non-fatal (best-effort hygiene)."""
+ if not os.path.isdir(CODE_BLOB_DIR):
+ return
+ for entry in os.listdir(CODE_BLOB_DIR):
+ if not entry.endswith(".zip"):
+ continue
+ sha = entry[: -len(".zip")]
+ if sha in referenced:
+ continue
+ try:
+ os.remove(os.path.join(CODE_BLOB_DIR, entry))
+ logger.debug("Lambda persistence: pruned orphan blob %s", sha)
+ except OSError as e:
+ logger.warning("Lambda persistence: failed to prune blob %s: %s", sha, e)
+
+
+def _externalize_code_zip(func: dict, referenced: set[str]) -> None:
+ """Replace ``func['code_zip']`` (bytes) with a blob ref in place, and
+ record the sha in ``referenced`` so the orphan sweep keeps the file."""
+ code = func.get("code_zip")
+ if not code:
+ return
+ if isinstance(code, bytes):
+ sha = _write_code_blob(code)
+ func["code_zip"] = {_BLOB_REF_KEY: sha}
+ referenced.add(sha)
+ elif isinstance(code, dict) and _BLOB_REF_KEY in code:
+ # Already a ref (e.g. ``get_state`` called twice without a restore
+ # in between). Keep the file referenced.
+ referenced.add(code[_BLOB_REF_KEY])
+ # Legacy inline base64 strings are not re-serialized here — they reach
+ # ``get_state`` only if a caller copied them directly into _functions
+ # without going through CreateFunction (no real code path does this).
+
+
+def _internalize_code_zip(func: dict) -> None:
+ """Replace a blob ref or legacy base64 string with bytes, in place."""
+ code = func.get("code_zip")
+ if not code:
+ return
+ if isinstance(code, dict) and _BLOB_REF_KEY in code:
+ func["code_zip"] = _read_code_blob(code[_BLOB_REF_KEY])
+ elif isinstance(code, str):
+ # Legacy persistence shape (inline base64).
+ func["code_zip"] = base64.b64decode(code)
+
+
def get_state():
- """Return JSON-serializable state. code_zip bytes are base64-encoded."""
+ """Return JSON-serializable Lambda state.
+
+ Function code (``code_zip``) is written to disk as content-addressed
+ blobs under ``${STATE_DIR}/lambda-blobs/`` and replaced in the returned
+ state with ``{"code_blob_ref": "<sha256>"}`` markers. Unreferenced blobs
+ from prior generations are pruned."""
from ministack.core.responses import AccountScopedDict
funcs = AccountScopedDict()
+ referenced: set[str] = set()
# Iterate _data directly to capture ALL accounts, not just current request context
for scoped_key, func in _functions._data.items():
f = copy.deepcopy(func)
- if f.get("code_zip") and isinstance(f["code_zip"], bytes):
- f["code_zip"] = base64.b64encode(f["code_zip"]).decode()
+ _externalize_code_zip(f, referenced)
for ver in f.get("versions", {}).values():
- if ver.get("code_zip") and isinstance(ver["code_zip"], bytes):
- ver["code_zip"] = base64.b64encode(ver["code_zip"]).decode()
+ _externalize_code_zip(ver, referenced)
funcs._data[scoped_key] = f
+ _prune_orphan_code_blobs(referenced)
return {
"functions": funcs,
"layers": copy.deepcopy(_layers),
@@ -256,19 +373,15 @@ def restore_state(data):
funcs = data.get("functions", {})
if isinstance(funcs, AccountScopedDict):
for scoped_key, func in funcs._data.items():
- if func.get("code_zip") and isinstance(func["code_zip"], str):
- func["code_zip"] = base64.b64decode(func["code_zip"])
+ _internalize_code_zip(func)
for ver in func.get("versions", {}).values():
- if ver.get("code_zip") and isinstance(ver["code_zip"], str):
- ver["code_zip"] = base64.b64decode(ver["code_zip"])
+ _internalize_code_zip(ver)
_functions._data[scoped_key] = func
else:
for name, func in funcs.items():
- if func.get("code_zip") and isinstance(func["code_zip"], str):
- func["code_zip"] = base64.b64decode(func["code_zip"])
+ _internalize_code_zip(func)
for ver in func.get("versions", {}).values():
- if ver.get("code_zip") and isinstance(ver["code_zip"], str):
- ver["code_zip"] = base64.b64decode(ver["code_zip"])
+ _internalize_code_zip(ver)
_functions[name] = func
_layers.update(data.get("layers", {}))
_esms.update(data.get("esms", {}))
diff --git a/tests/test_lambda.py b/tests/test_lambda.py
index cecdbfbb..fdb40149 100644
--- a/tests/test_lambda.py
+++ b/tests/test_lambda.py
@@ -4510,6 +4510,194 @@ def test_lambda_durable_persistence_round_trip():
lambda_durable.restore_state(original)
+# ---------------------------------------------------------------------------
+# Lambda code_zip persistence — content-addressed blob storage.
+# code_zip bytes are written to ${STATE_DIR}/lambda-blobs/{sha256}.zip; the
+# returned state holds only the sha reference. The in-memory _functions
+# shape is unchanged (still bytes after restore) so invoke / update / delete
+# paths see no difference.
+# ---------------------------------------------------------------------------
+
+
+def _make_lambda_record(name: str, code_zip: bytes, versions: dict | None = None) -> dict:
+ """Build a _functions entry matching what CreateFunction stores."""
+ return {
+ "config": {
+ "FunctionName": name,
+ "FunctionArn": f"arn:aws:lambda:us-east-1:000000000000:function:{name}",
+ "Runtime": "python3.12",
+ "Handler": "index.handler",
+ },
+ "code_zip": code_zip,
+ "versions": versions or {},
+ "next_version": 1,
+ "tags": {},
+ "policy": {"Version": "2012-10-17", "Id": "default", "Statement": []},
+ }
+
+
+def lambda_svc_isolated(tmp_path, monkeypatch):
+ """Snapshot lambda_svc state and redirect its blob storage to tmp_path.
+ Restores the original state at teardown so tests don't pollute the
+ shared in-process module that serves the running ministack."""
+ from ministack.services import lambda_svc
+
+ monkeypatch.setattr(lambda_svc, "CODE_BLOB_DIR", str(tmp_path / "lambda-blobs"))
+ original = lambda_svc.get_state()
+ try:
+ lambda_svc._functions._data.clear()
+ yield lambda_svc, tmp_path / "lambda-blobs"
+ finally:
+ lambda_svc._functions._data.clear()
+ lambda_svc.restore_state(original)
+
+
+def test_lambda_code_zip_round_trip_through_blob_storage(lambda_svc_isolated):
+ """Bytes survive an exact get_state → clear → restore_state round-trip."""
+ svc, _ = lambda_svc_isolated
+ code = b"\x89PNG\r\n\x1a\n" + b"\x00" * 1000 # non-UTF8 bytes
+ svc._functions["fn"] = _make_lambda_record("fn", code)
+
+ state = svc.get_state()
+ svc._functions._data.clear()
+ svc.restore_state(state)
+
+ restored = svc._functions._data[("000000000000", "fn")]
+ assert restored["code_zip"] == code
+ assert isinstance(restored["code_zip"], bytes)
+
+
+def test_lambda_get_state_replaces_code_zip_with_blob_ref(lambda_svc_isolated):
+ """The returned state must not inline bytes or base64 — only a sha ref."""
+ import hashlib
+ svc, blob_dir = lambda_svc_isolated
+ code = b"def handler(event, ctx): return 'ok'\n" * 256
+ svc._functions["fn"] = _make_lambda_record("fn", code)
+
+ state = svc.get_state()
+ fn_state = state["functions"]._data[("000000000000", "fn")]
+
+ assert fn_state["code_zip"] == {"code_blob_ref": hashlib.sha256(code).hexdigest()}
+ blob_path = blob_dir / f"{hashlib.sha256(code).hexdigest()}.zip"
+ assert blob_path.exists()
+ assert blob_path.read_bytes() == code
+
+
+def test_lambda_per_version_code_zip_also_externalized(lambda_svc_isolated):
+ """PublishVersion-created versions also store code_zip externally."""
+ import hashlib
+ svc, _ = lambda_svc_isolated
+ v1, v2 = b"v1 body", b"v2 body"
+ svc._functions["fn"] = _make_lambda_record(
+ "fn", v2, versions={"1": {"code_zip": v1, "config": {"Version": "1"}}}
+ )
+
+ state = svc.get_state()
+ fn_state = state["functions"]._data[("000000000000", "fn")]
+ assert fn_state["versions"]["1"]["code_zip"] == {
+ "code_blob_ref": hashlib.sha256(v1).hexdigest()
+ }
+
+ svc._functions._data.clear()
+ svc.restore_state(state)
+ restored = svc._functions._data[("000000000000", "fn")]
+ assert restored["code_zip"] == v2
+ assert restored["versions"]["1"]["code_zip"] == v1
+
+
+def test_lambda_identical_code_dedups_to_single_blob(lambda_svc_isolated):
+ """Content-addressing means two functions with identical bytes share one
+ file. Important when the deps-bundled-into-every-zip pattern produces
+ many functions with byte-identical layer payloads."""
+ svc, blob_dir = lambda_svc_isolated
+ code = b"shared body across two functions"
+ svc._functions["fn-a"] = _make_lambda_record("fn-a", code)
+ svc._functions["fn-b"] = _make_lambda_record("fn-b", code)
+
+ svc.get_state()
+
+ files = sorted(blob_dir.iterdir())
+ assert len(files) == 1, [f.name for f in files]
+
+
+def test_lambda_legacy_inline_base64_persistence_still_loads(lambda_svc_isolated):
+ """Pre-existing lambda.json files (written before content-addressed
+ storage) stored code_zip as inline base64. restore_state must accept
+ that shape so an in-place upgrade requires no migration step."""
+ import base64 as _b64
+ from ministack.core.responses import AccountScopedDict
+ svc, _ = lambda_svc_isolated
+
+ code = b"legacy persisted bytes"
+ legacy = {
+ "functions": AccountScopedDict(),
+ "layers": AccountScopedDict(),
+ "esms": AccountScopedDict(),
+ "function_urls": AccountScopedDict(),
+ "kinesis_positions": {},
+ "dynamodb_stream_positions": {},
+ }
+ legacy["functions"]._data[("000000000000", "old-fn")] = {
+ "config": {"FunctionName": "old-fn"},
+ "code_zip": _b64.b64encode(code).decode(),
+ "versions": {},
+ }
+
+ svc.restore_state(legacy)
+
+ restored = svc._functions._data[("000000000000", "old-fn")]
+ assert restored["code_zip"] == code
+
+
+def test_lambda_missing_blob_degrades_without_aborting_restore(lambda_svc_isolated):
+ """If a blob file is missing (corrupted volume / partial mount), restore
+ must downgrade the affected function (code_zip=None) and continue,
+ rather than raise and prevent the whole server from starting."""
+ from ministack.core.responses import AccountScopedDict
+ svc, _ = lambda_svc_isolated
+
+ state = {
+ "functions": AccountScopedDict(),
+ "layers": AccountScopedDict(),
+ "esms": AccountScopedDict(),
+ "function_urls": AccountScopedDict(),
+ "kinesis_positions": {},
+ "dynamodb_stream_positions": {},
+ }
+ state["functions"]._data[("000000000000", "orphan")] = {
+ "config": {"FunctionName": "orphan"},
+ "code_zip": {"code_blob_ref": "0" * 64}, # sha pointing at nothing
+ "versions": {},
+ }
+
+ svc.restore_state(state)
+
+ assert svc._functions._data[("000000000000", "orphan")]["code_zip"] is None
+
+
+def test_lambda_get_state_prunes_orphan_blobs(lambda_svc_isolated):
+ """When a function's code is updated, the previous generation's blob
+ becomes orphan. The next get_state sweeps it. Without this, repeated
+ UpdateFunctionCode (e.g. iterative sandbox redeploys) leaks blob files
+ across restarts."""
+ import hashlib
+ svc, blob_dir = lambda_svc_isolated
+
+ old_code = b"old code"
+ new_code = b"new code"
+ svc._functions["fn"] = _make_lambda_record("fn", old_code)
+ svc.get_state() # writes blob(old_code)
+ assert (blob_dir / f"{hashlib.sha256(old_code).hexdigest()}.zip").exists()
+
+ # Simulate UpdateFunctionCode.
+ svc._functions["fn"]["code_zip"] = new_code
+ svc.get_state() # writes blob(new_code), should remove blob(old_code)
+
+ assert (blob_dir / f"{hashlib.sha256(new_code).hexdigest()}.zip").exists()
+ assert not (blob_dir / f"{hashlib.sha256(old_code).hexdigest()}.zip").exists()
+
+
def test_lambda_durable_event_wrapped_with_sdk_fields(lam):
"""A durable invocation's event payload is wrapped with the fields the
aws-durable-execution-sdk-python SDK reads from the Lambda event:
diff --git a/tests/test_ministack.py b/tests/test_ministack.py
index e950158d..85c30dc2 100644
--- a/tests/test_ministack.py
+++ b/tests/test_ministack.py
@@ -364,23 +364,35 @@ def test_ministack_persist_s3_logging_accelerate_request_payment_roundtrip():
_s3._bucket_request_payment_config.pop("persist-log-bkt")
def test_ministack_persist_lambda_roundtrip():
+ import hashlib
+ import os
from ministack.services import lambda_svc as _lam
+ code = b"fake-zip-bytes"
+ sha = hashlib.sha256(code).hexdigest()
_lam._functions["persist-fn"] = {
"config": {"FunctionName": "persist-fn", "Runtime": "python3.11"},
- "code_zip": b"fake-zip-bytes",
+ "code_zip": code,
"versions": {},
"next_version": 1,
}
- state = _lam.get_state()
- assert "functions" in state
- # code_zip should be base64-encoded in state
- assert isinstance(state["functions"]["persist-fn"]["code_zip"], str)
- _lam._functions.pop("persist-fn")
- _lam.restore_state(state)
- assert "persist-fn" in _lam._functions
- # code_zip should be decoded back to bytes
- assert _lam._functions["persist-fn"]["code_zip"] == b"fake-zip-bytes"
- _lam._functions.pop("persist-fn")
+ try:
+ state = _lam.get_state()
+ assert "functions" in state
+ # code_zip is replaced with a content-addressed blob reference in state;
+ # the bytes themselves live under ${STATE_DIR}/lambda-blobs/{sha}.zip.
+ assert state["functions"]["persist-fn"]["code_zip"] == {"code_blob_ref": sha}
+ _lam._functions.pop("persist-fn")
+ _lam.restore_state(state)
+ assert "persist-fn" in _lam._functions
+ # code_zip is restored back to bytes.
+ assert _lam._functions["persist-fn"]["code_zip"] == code
+ finally:
+ _lam._functions.pop("persist-fn", None)
+ # Clean up the blob file written as a side effect of get_state.
+ try:
+ os.remove(os.path.join(_lam.CODE_BLOB_DIR, f"{sha}.zip"))
+ except OSError:
+ pass
def test_ministack_persist_rds_roundtrip():
from ministack.services import rds as _rds