A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: make the pickling mechanism in mo.cache more robust (#8761)
Dylan Madisettide6fabf608c2
diff --git a/marimo/_save/encode.py b/marimo/_save/encode.pynew file mode 100644index 0000000000..38efdf742b--- /dev/null+++ b/marimo/_save/encode.py@@ -0,0 +1,172 @@+# Copyright 2026 Marimo. All rights reserved.+"""Deterministic encoding of Python values to bytes for content-addressed hashing.++This module owns the conversion of Python objects (primitives, tensors,+containers, arbitrary picklable objects) into canonical byte sequences.+"""++from __future__ import annotations++import hashlib+import io+import pickle+import struct+from typing import TYPE_CHECKING, Any++from marimo._dependencies.dependencies import DependencyManager+from marimo._runtime.primitives import is_data_primitive, is_primitive++if TYPE_CHECKING:+ from collections.abc import Iterable++ # Union[torch.Tensor, jax.numpy.ndarray,+ # np.ndarray, scipy.sparse.spmatrix]+ Tensor = Any+++def type_sign(value: bytes, label: str) -> bytes:+ # Appending all strings with a key disambiguates it from other types. e.g.+ # when the string value is the same as a float pack, or is the literal+ # ":none". If our content strings take the form: integrity + delimiter then+ # these types of collisions become very hard.+ #+ # Note that this does not fully protect against cache poisoning, as an+ # attacker can override python internals to provide a matched hash. A key+ # signed cache result is the only way to properly protect against this.+ #+ # Additionally, (less meaningful, but still possible)- a byte collision can+ # be manufactured by choosing data so long that the length of the data acts+ # as the data injection.+ #+ # TODO: Benchmark something like `sha1 (integrity) + delimiter`, this+ # method is chosen because it was assumed to be fast, but might be slow+ # with a copy of large data.+ return b"".join([value, bytes(len(value)), bytes(":" + label, "utf-8")])+++def iterable_sign(value: Iterable[Any], label: str) -> bytes:+ values = list(value)+ return b"".join(+ [b"".join(values), bytes(len(values)), bytes(":" + label, "utf-8")]+ )+++def standardize_tensor(tensor: Tensor) -> Tensor:+ if (+ hasattr(tensor, "__array__")+ or hasattr(tensor, "toarray")+ or hasattr(tensor, "__array_interface__")+ ):+ DependencyManager.numpy.require("to access data buffer for hashing.")+ import numpy++ if not hasattr(tensor, "__array_interface__"):+ # Capture those sparse cases+ if hasattr(tensor, "toarray"):+ tensor = tensor.toarray()+ # As array should not perform copy+ return numpy.asarray(tensor)+ raise ValueError(+ f"Expected a data primitive object, but got {type(tensor)} instead."+ "This maybe is an internal marimo issue. Please report to "+ "https://github.com/marimo-team/marimo/issues."+ )+++def _contiguous_tensor_bytes(data: Tensor) -> memoryview:+ """Return a contiguous uint8 view of a tensor/array."""+ data = standardize_tensor(data)+ # From joblib.hashing+ if data.shape == ():+ # 0d arrays need to be flattened because viewing them as bytes+ # raises a ValueError exception.+ data_c_contiguous = data.flatten()+ elif data.flags.c_contiguous:+ data_c_contiguous = data+ elif data.flags.f_contiguous:+ data_c_contiguous = data.T+ else:+ # Cater for non-single-segment arrays, this creates a copy, and thus+ # alleviates this issue. Note: There might be a more efficient way of+ # doing this, check for joblib updates.+ data_c_contiguous = data.flatten()+ return memoryview(data_c_contiguous.view("uint8"))+++def data_to_buffer(data: Tensor) -> bytes:+ return type_sign(_contiguous_tensor_bytes(data), "data")+++def primitive_to_bytes(value: Any) -> bytes:+ if value is None:+ return b":none"+ if isinstance(value, str):+ return type_sign(bytes(f"{value}", "utf-8"), "str")+ if isinstance(value, float):+ return type_sign(struct.pack("d", value), "float")+ if isinstance(value, int):+ return type_sign(struct.pack("q", value), "int")+ if isinstance(value, tuple):+ return iterable_sign(map(primitive_to_bytes, value), "tuple")+ return type_sign(bytes(value), "bytes")+++def common_container_to_bytes(value: Any) -> bytes:+ visited: dict[int, int] = {}++ def recurse_container(value: Any) -> bytes:+ if id(value) in visited:+ return type_sign(bytes(visited[id(value)]), "id")+ if isinstance(value, dict):+ visited[id(value)] = len(visited)+ return iterable_sign(+ map(recurse_container, sorted(value.items())), "dict"+ )+ if isinstance(value, list):+ visited[id(value)] = len(visited)+ return iterable_sign(map(recurse_container, value), "list")+ if isinstance(value, set):+ visited[id(value)] = len(visited)+ return iterable_sign(map(recurse_container, sorted(value)), "set")+ # Tuple may be only data primitive, not fully primitive.+ if isinstance(value, tuple):+ return iterable_sign(map(recurse_container, value), "tuple")++ if is_primitive(value):+ return primitive_to_bytes(value)+ return data_to_buffer(value)++ return recurse_container(value)+++def attempt_signed_bytes(value: bytes, label: str) -> bytes:+ # Prevents hash collisions like:+ # >>> fib(1)+ # >>> s, _ = state(1)+ # >>> fib(s)+ # ^ would be a cache hit as is even though fib(s) would fail by+ # itself+ try:+ return type_sign(common_container_to_bytes(value), label)+ # Fallback to raw state for eval in content hash.+ except (TypeError, ValueError):+ return value+++def deterministic_dumps(obj: Any, hash_type: str) -> bytes:+ """``pickle.dumps`` replacement that produces more deterministic bytes."""+ from marimo._save.stubs import maybe_get_custom_stub++ class _ContentHashPickler(pickle.Pickler):+ def reducer_override(self, obj: Any) -> Any:+ if stub := maybe_get_custom_stub(obj):+ return (bytes, (stub.to_bytes(),))+ if not is_primitive(obj) and is_data_primitive(obj):+ h = hashlib.new(hash_type, usedforsecurity=False)+ h.update(_contiguous_tensor_bytes(obj))+ return (bytes, (h.digest(),))+ return NotImplemented++ buf = io.BytesIO()+ _ContentHashPickler(buf).dump(obj)+ return buf.getvalue()diff --git a/marimo/_save/hash.py b/marimo/_save/hash.pyindex 5a05dfd068..d2ff36e74a 100644--- a/marimo/_save/hash.py+++ b/marimo/_save/hash.py@@ -6,7 +6,6 @@ import base64import dataclassesimport hashlibimport inspect-import structimport sysimport typesfrom typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional@@ -18,7 +17,6 @@ from marimo._ast.variables import (unmangle_local,)from marimo._ast.visitor import ImportData, Name, ScopedVisitor-from marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins.ui._core.ui_element import UIElementfrom marimo._runtime.context import ContextNotInitializedError, get_contextfrom marimo._runtime.dataflow import induced_subgraph@@ -36,11 +34,18 @@ from marimo._runtime.side_effect import CellHash, SideEffectfrom marimo._runtime.state import SetFunctor, Statefrom marimo._runtime.watch._path import PathStatefrom marimo._save.cache import Cache, CacheType+from marimo._save.encode import (+ attempt_signed_bytes,+ common_container_to_bytes,+ data_to_buffer,+ deterministic_dumps,+ primitive_to_bytes,+ type_sign,+)from marimo._save.stubs import maybe_get_custom_stubfrom marimo._types.ids import CellId_tif TYPE_CHECKING:- from collections.abc import Iterablefrom types import CodeTypefrom marimo._ast.cell import CellImpl@@ -48,11 +53,6 @@ if TYPE_CHECKING:from marimo._runtime.dataflow import DirectedGraphfrom marimo._save.loaders import Loader- # Union[list, torch.Tensor, jax.numpy.ndarray,- # np.ndarray, scipy.sparse.spmatrix]- Tensor = Any--# Default hash type is generally inconsequential, there may be implications of# malicious hash collision or performance. Malicious hash collision can be# mitigated with a signed cache, and performance is neligible compared to the@@ -169,130 +169,6 @@ def hash_cell_execution(return hash_cell_group(ancestors, graph, hash_type)-def standardize_tensor(tensor: Tensor) -> Optional[Tensor]:- if (- hasattr(tensor, "__array__")- or hasattr(tensor, "toarray")- or hasattr(tensor, "__array_interface__")- ):- DependencyManager.numpy.require("to access data buffer for hashing.")- import numpy-- if not hasattr(tensor, "__array_interface__"):- # Capture those sparse cases- if hasattr(tensor, "toarray"):- tensor = tensor.toarray()- # As array should not perform copy- return numpy.asarray(tensor)- raise ValueError(- f"Expected a data primitive object, but got {type(tensor)} instead."- "This maybe is an internal marimo issue. Please report to "- "https://github.com/marimo-team/marimo/issues."- )---def type_sign(value: bytes, label: str) -> bytes:- # Appending all strings with a key disambiguates it from other types. e.g.- # when the string value is the same as a float pack, or is the literal- # ":none". If our content strings take the form: integrity + delimiter then- # these types of collisions become very hard.- #- # Note that this does not fully protect against cache poisoning, as an- # attacker can override python internals to provide a matched hash. A key- # signed cache result is the only way to properly protect against this.- #- # Additionally, (less meaningful, but still possible)- a byte collision can- # be manufactured by choosing data so long that the length of the data acts- # as the data injection.- #- # TODO: Benchmark something like `sha1 (integrity) + delimiter`, this- # method is chosen because it was assumed to be fast, but might be slow- # with a copy of large data.- return b"".join([value, bytes(len(value)), bytes(":" + label, "utf-8")])---def iterable_sign(value: Iterable[Any], label: str) -> bytes:- values = list(value)- return b"".join(- [b"".join(values), bytes(len(values)), bytes(":" + label, "utf-8")]- )---def primitive_to_bytes(value: Any) -> bytes:- if value is None:- return b":none"- if isinstance(value, str):- return type_sign(bytes(f"{value}", "utf-8"), "str")- if isinstance(value, float):- return type_sign(struct.pack("d", value), "float")- if isinstance(value, int):- return type_sign(struct.pack("q", value), "int")- if isinstance(value, tuple):- return iterable_sign(map(primitive_to_bytes, value), "tuple")- return type_sign(bytes(value), "bytes")---def common_container_to_bytes(value: Any) -> bytes:- visited: dict[int, int] = {}-- def recurse_container(value: Any) -> bytes:- if id(value) in visited:- return type_sign(bytes(visited[id(value)]), "id")- if isinstance(value, dict):- visited[id(value)] = len(visited)- return iterable_sign(- map(recurse_container, sorted(value.items())), "dict"- )- if isinstance(value, list):- visited[id(value)] = len(visited)- return iterable_sign(map(recurse_container, value), "list")- if isinstance(value, set):- visited[id(value)] = len(visited)- return iterable_sign(map(recurse_container, sorted(value)), "set")- # Tuple may be only data primitive, not fully primitive.- if isinstance(value, tuple):- return iterable_sign(map(recurse_container, value), "tuple")-- if is_primitive(value):- return primitive_to_bytes(value)- return data_to_buffer(value)-- return recurse_container(value)---def data_to_buffer(data: Tensor) -> bytes:- data = standardize_tensor(data)- # From joblib.hashing- if data.shape == ():- # 0d arrays need to be flattened because viewing them as bytes- # raises a ValueError exception.- data_c_contiguous = data.flatten()- elif data.flags.c_contiguous:- data_c_contiguous = data- elif data.flags.f_contiguous:- data_c_contiguous = data.T- else:- # Cater for non-single-segment arrays, this creates a copy, and thus- # alleviates this issue. Note: There might be a more efficient way of- # doing this, check for joblib updates.- data_c_contiguous = data.flatten()- return type_sign(memoryview(data_c_contiguous.view("uint8")), "data")---def attempt_signed_bytes(value: bytes, label: str) -> bytes:- # Prevents hash collisions like:- # >>> fib(1)- # >>> s, _ = state(1)- # >>> fib(s)- # ^ would be a cache hit as is even though fib(s) would fail by- # itself- try:- return type_sign(common_container_to_bytes(value), label)- # Fallback to raw state for eval in content hash.- except (TypeError, ValueError):- return value--def get_and_update_context_from_scope(scope: dict[str, Any],scope_refs: Optional[set[Name]] = None,@@ -616,7 +492,9 @@ class BlockHasher:for ref in unhashable:try:- _hashed = pickle.dumps(scope[ref])+ _hashed = deterministic_dumps(+ scope[ref], self.hash_alg.name+ )content_serialization[ref] = type_sign(_hashed, "pickle")refs.remove(ref)except (pickle.PicklingError, TypeError) as e:diff --git a/tests/_save/test_encode.py b/tests/_save/test_encode.pynew file mode 100644index 0000000000..958c9cae23--- /dev/null+++ b/tests/_save/test_encode.py@@ -0,0 +1,81 @@+# Copyright 2026 Marimo. All rights reserved.++from __future__ import annotations++import array+import pickle+from typing import Any++from marimo._save.encode import deterministic_dumps+++class _ArrayWithSet:+ """Array-like whose pickle is non-deterministic across processes.++ The ``_tags`` set has hash-randomized iteration order across Python+ processes (PYTHONHASHSEED), so ``pickle.dumps`` produces different bytes+ on each run. But since this class has ``__array__``, ``_ContentHashPickler``+ replaces it with a deterministic content hash of the array data, discarding+ the non-deterministic set entirely.+ """++ def __init__(self, data: list[float], tags: set) -> None:+ self._data = array.array("f", data)+ self._tags = tags++ def __array__(self, dtype: Any = None) -> Any:+ import numpy as np++ return np.frombuffer(self._data, dtype=np.float32)+++def test_reducer_fires_replaces_with_content_hash() -> None:+ """reducer_override must replace data primitives, not pass through to pickle."""+ obj = _ArrayWithSet([1.0, 2.0, 3.0], tags={"a", "b", "c"})++ raw = pickle.dumps(obj)+ det = deterministic_dumps(obj, "sha256")++ assert raw != det+++def test_deterministic_for_same_data_different_set() -> None:+ """Two objects with same array data but different set attrs must hash the same.++ This mirrors the cross-process case: PYTHONHASHSEED changes set iteration+ order, so ``pickle.dumps`` differs between processes. ``deterministic_dumps``+ must not — it is content-addressed on the array data only.+ """+ data = list(range(10))++ obj1 = _ArrayWithSet(data, tags={"x", "y", "z"})+ obj2 = _ArrayWithSet(data, tags={"p", "q", "r"})++ assert pickle.dumps(obj1) != pickle.dumps(obj2)+ assert deterministic_dumps(obj1, "sha256") == deterministic_dumps(+ obj2, "sha256"+ )+++def test_different_array_data_produces_different_hash() -> None:+ """Different array data must produce different deterministic hashes."""+ obj1 = _ArrayWithSet([0.0] * 10, tags={"a"})+ obj2 = _ArrayWithSet([1.0] * 10, tags={"a"})++ assert deterministic_dumps(obj1, "sha256") != deterministic_dumps(+ obj2, "sha256"+ )+++def test_plain_objects_pass_through() -> None:+ """Objects without data primitives should pickle normally."""+ obj = {"key": "value", "n": 42, "nested": [1, 2, 3]}+ assert deterministic_dumps(obj, "sha256") == pickle.dumps(obj)+++def test_0d_array() -> None:+ """Scalar (0-d) arrays are a special code path in _contiguous_tensor_bytes."""+ obj = _ArrayWithSet([3.14], tags={"x"})+ det1 = deterministic_dumps(obj, "sha256")+ det2 = deterministic_dumps(obj, "sha256")+ assert det1 == det2