A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: prevent UI / state from being miskeyed for hash construction (#10447)
Dylan Madisettic9170147234a
diff --git a/marimo/_save/encode.py b/marimo/_save/encode.pyindex 7236c138f0..9953270d91 100644--- a/marimo/_save/encode.py+++ b/marimo/_save/encode.py@@ -131,6 +131,11 @@ def common_container_to_bytes(value: Any) -> bytes:# Tuple may be only data primitive, not fully primitive.if isinstance(value, tuple):return iterable_sign(map(recurse_container, value), "tuple")+ # bytearray is mutable so not in PRIMITIVES, but its bytes are a+ # deterministic content key; sign under a distinct label so it does not+ # collide with an equal-content bytes value.+ if isinstance(value, bytearray):+ return type_sign(bytes(value), "bytearray")if is_primitive(value):return primitive_to_bytes(value)diff --git a/marimo/_save/hash.py b/marimo/_save/hash.pyindex 2ad2b9b13d..1fb50ee032 100644--- a/marimo/_save/hash.py+++ b/marimo/_save/hash.py@@ -297,7 +297,7 @@ class BlockHasher:if not external:ctx = get_and_update_context_from_scope(scope)refs, _, stateful_refs = self.extract_ref_state_and_normalize_scope(- refs, scope, ctx+ refs, scope, ctx, hash_type)self.stateful_refs = stateful_refs@@ -562,11 +562,35 @@ class BlockHasher:missing.add(ref)return _refs, missing+ def _signed_stateful_bytes(+ self, value: Any, label: str, hash_type: str = DEFAULT_HASH+ ) -> bytes:+ """Content bytes for a state/UI value, pickling as a last resort."""+ signed = attempt_signed_bytes(value, label)+ if isinstance(signed, bytes):+ return signed++ # A registered custom stub defines the object's bytes.+ if (stub := maybe_get_custom_stub(value)) is not None:+ return type_sign(stub.to_bytes(), "stub")++ # Fallback for objects mapped to UI/State.+ try:+ return type_sign(deterministic_dumps(value, hash_type), "pickle")+ except Exception as e:+ # If stateful and unpicklable, the value can't be keyed, so raise+ # instead of miskeying it.+ raise TypeError(+ f"Cached cell depends on a {label} value that is neither "+ f"content-addressable nor picklable: {type(value).__name__}."+ ) from e+def extract_ref_state_and_normalize_scope(self,refs: set[Name],scope: dict[str, Any],ctx: RuntimeContext | None = None,+ hash_type: str = DEFAULT_HASH,) -> SerialRefs:"""Preprocess the scope and references, and extract state references.@@ -581,6 +605,7 @@ class BlockHasher:refs: A set of reference names.scope: A dictionary representing the current scope.ctx: An optional runtime context for stateful lookup.+ hash_type: The hash algorithm used to serialize stateful values.Returns:SerialRefs tuple containing the following elements:@@ -620,7 +645,9 @@ class BlockHasher:repr(value).encode(), "pathstate")else:- scope[ref] = attempt_signed_bytes(value(), "state")+ scope[ref] = self._signed_stateful_bytes(+ value(), "state", hash_type+ )if ctx:for state_name in ctx.state_registry.bound_names(value):scope[state_name] = scope[ref]@@ -634,7 +661,9 @@ class BlockHasher:if ui is not None and (ref not in scope or isinstance(scope[ref], UIElement)):- scope[ref] = attempt_signed_bytes(ui.value, "ui")+ scope[ref] = self._signed_stateful_bytes(+ ui.value, "ui", hash_type+ )if ctx:for ui_name in ctx.ui_element_registry.bound_names(ui._id):scope[ui_name] = scope[ref]@@ -883,7 +912,7 @@ class BlockHasher:# Need to run extract again for the expanded ref set.refs, _, stateful_refs = self.extract_ref_state_and_normalize_scope(- refs, scope, ctx+ refs, scope, ctx, self.hash_alg.name)# Attempt content hash again on the extracted stateful refs.# Do not pass ctx — stateful values can change between cells,@@ -1198,7 +1227,7 @@ def content_cache_attempt_from_base(hasher = BlockHasher.from_parent(previous_block)ctx = get_and_update_context_from_scope(scope, required_refs)refs, _, stateful_refs = hasher.extract_ref_state_and_normalize_scope(- refs, scope, ctx+ refs, scope, ctx, hasher.hash_alg.name)refs, _content, tmp_stateful_refs = hasher.collect_for_content_hash(diff --git a/tests/_save/test_encode.py b/tests/_save/test_encode.pyindex a364091643..362bf4d062 100644--- a/tests/_save/test_encode.py+++ b/tests/_save/test_encode.py@@ -9,7 +9,7 @@ from typing import Anyimport pytestfrom marimo._dependencies.dependencies import DependencyManager-from marimo._save.encode import deterministic_dumps+from marimo._save.encode import common_container_to_bytes, deterministic_dumpsHAS_PANDAS = DependencyManager.pandas.has()HAS_NUMPY = DependencyManager.numpy.has()@@ -122,3 +122,18 @@ def test_dataframe_same_data_same_hash() -> None:assert deterministic_dumps(df1, hash_type="sha256") == deterministic_dumps(df2, hash_type="sha256")+++def test_bytearray_encodes_to_deterministic_bytes() -> None:+ """A bytearray encodes to deterministic bytes that vary by content."""+ a = common_container_to_bytes(bytearray(b"abc"))+ assert type(a) is bytes+ assert a == common_container_to_bytes(bytearray(b"abc"))+ assert a != common_container_to_bytes(bytearray(b"xyz"))+++def test_bytearray_does_not_collide_with_equal_bytes() -> None:+ """bytearray and bytes of equal content sign under distinct labels."""+ assert common_container_to_bytes(bytearray(b"abc")) != (+ common_container_to_bytes(b"abc")+ )diff --git a/tests/_save/test_hash.py b/tests/_save/test_hash.pyindex 44f6fe5a4e..88567b80f5 100644--- a/tests/_save/test_hash.py+++ b/tests/_save/test_hash.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import dataclassesimport osimport subprocessimport sys@@ -15,6 +16,7 @@ from marimo._ast.app import Appfrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime.commands import ExecuteStaleCellsCommandfrom marimo._runtime.runtime import Kernel+from marimo._save.hash import BlockHasherfrom tests.conftest import ExecReqProvider@@ -2813,3 +2815,80 @@ hash_val = cached_query._last_hashf"Expected different hashes when decorator ref changes, "f"got {first_hash} == {second_hash}")++[email protected](frozen=True)+class _Selection:+ """Stand-in for a UI value (e.g. mo.ui.matplotlib selection) that is+ picklable but neither a primitive nor a data-primitive."""++ vertices: tuple[tuple[float, float], ...]+++def test_signed_stateful_bytes_content_varies() -> None:+ # A stateful value that is not content-addressable must still hash by its+ # content (pickle last-resort), so a consumer re-keys when it changes.+ hasher = BlockHasher.__new__(BlockHasher)+ a = hasher._signed_stateful_bytes(_Selection(((0.0, 0.0),)), "ui")+ b = hasher._signed_stateful_bytes(_Selection(((1.0, 2.0),)), "ui")+ assert isinstance(a, bytes)+ assert isinstance(b, bytes)+ assert a != b+ # Same value -> same bytes (deterministic).+ assert a == hasher._signed_stateful_bytes(_Selection(((0.0, 0.0),)), "ui")+++def test_signed_stateful_bytes_uses_custom_stub() -> None:+ # A value with a registered custom stub hashes by the stub's canonical+ # bytes (what the cache stores it as), not a generic pickle.+ from marimo._save.encode import type_sign+ from marimo._save.stubs import CUSTOM_STUBS, CustomStub, register_stub++ @dataclasses.dataclass+ class _Model:+ payload: bytes++ class _ModelStub(CustomStub):+ __slots__ = ("payload",)++ def __init__(self, obj: Any) -> None:+ self.payload = obj.payload++ def load(self, glbls: dict[str, Any]) -> Any:+ del glbls+ return _Model(self.payload)++ @staticmethod+ def get_type() -> type:+ return _Model++ def to_bytes(self) -> bytes:+ return self.payload++ register_stub(_Model, _ModelStub)+ try:+ hasher = BlockHasher.__new__(BlockHasher)+ out = hasher._signed_stateful_bytes(_Model(b"abc"), "ui")+ assert out == type_sign(b"abc", "stub")+ # Distinct stub bytes -> distinct key.+ assert out != hasher._signed_stateful_bytes(_Model(b"xyz"), "ui")+ finally:+ CUSTOM_STUBS.pop(_Model, None)+++def test_signed_stateful_bytes_bytearray_is_signed_bytes() -> None:+ # A bytearray value must hash to signed, immutable bytes (not a raw,+ # mutable bytearray), and vary by content.+ hasher = BlockHasher.__new__(BlockHasher)+ a = hasher._signed_stateful_bytes(bytearray(b"abc"), "ui")+ b = hasher._signed_stateful_bytes(bytearray(b"xyz"), "ui")+ assert type(a) is bytes+ assert type(b) is bytes+ assert a != b+ assert a == hasher._signed_stateful_bytes(bytearray(b"abc"), "ui")+++def test_signed_stateful_bytes_unpicklable_raises() -> None:+ hasher = BlockHasher.__new__(BlockHasher)+ with pytest.raises(TypeError, match="neither"):+ hasher._signed_stateful_bytes(lambda: None, "ui")