A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
import array
import pickle
from typing import Any
import pytest
from marimo._dependencies.dependencies import DependencyManager
from marimo._save.encode import (
attempt_signed_bytes,
common_container_to_bytes,
data_to_buffer,
deterministic_dumps,
)
HAS_PANDAS = DependencyManager.pandas.has()
HAS_NUMPY = DependencyManager.numpy.has()
class _ArrayWithSet:
"""Array-like whose pickle is non-deterministic across processes.
The `_tags` set has hash-randomized iteration order across Python
processes (PYTHONHASHSEED), so `pickle.dumps` produces different bytes
on each run. But since this class has `__array__`, `_ContentHashPickler`
replaces it with a deterministic content hash of the array data, discarding
the non-deterministic set entirely.
"""
def __init__(self, data: list[float], tags: set) -> None:
self._data = array.array("f", data)
self._tags = tags
def __array__(self, dtype: Any = None) -> Any:
import numpy as np
return np.frombuffer(self._data, dtype=np.float32)
def test_reducer_fires_replaces_with_content_hash() -> None:
"""reducer_override must replace data primitives, not pass through to pickle."""
obj = _ArrayWithSet([1.0, 2.0, 3.0], tags={"a", "b", "c"})
raw = pickle.dumps(obj)
det = deterministic_dumps(obj, "sha256")
assert raw != det
def test_deterministic_for_same_data_different_set() -> None:
"""Two objects with same array data but different set attrs must hash the same.
This mirrors the cross-process case: PYTHONHASHSEED changes set iteration
order, so `pickle.dumps` differs between processes. `deterministic_dumps`
must not — it is content-addressed on the array data only.
"""
data = list(range(10))
obj1 = _ArrayWithSet(data, tags={"x", "y", "z"})
obj2 = _ArrayWithSet(data, tags={"p", "q", "r"})
assert pickle.dumps(obj1) != pickle.dumps(obj2)
assert deterministic_dumps(obj1, "sha256") == deterministic_dumps(
obj2, "sha256"
)
def test_different_array_data_produces_different_hash() -> None:
"""Different array data must produce different deterministic hashes."""
obj1 = _ArrayWithSet([0.0] * 10, tags={"a"})
obj2 = _ArrayWithSet([1.0] * 10, tags={"a"})
assert deterministic_dumps(obj1, "sha256") != deterministic_dumps(
obj2, "sha256"
)
def test_plain_objects_pass_through() -> None:
"""Objects without data primitives should pickle normally."""
obj = {"key": "value", "n": 42, "nested": [1, 2, 3]}
assert deterministic_dumps(obj, "sha256") == pickle.dumps(obj)
def test_0d_array() -> None:
"""Scalar (0-d) arrays are a special code path in _contiguous_tensor_bytes."""
obj = _ArrayWithSet([3.14], tags={"x"})
det1 = deterministic_dumps(obj, "sha256")
det2 = deterministic_dumps(obj, "sha256")
assert det1 == det2
@pytest.mark.skipif(
not HAS_PANDAS or not HAS_NUMPY,
reason="pandas and numpy are required",
)
def test_dataframe_with_integer_columns() -> None:
"""Regression: pd.DataFrame(np.random.randn(3, 3)) has integer column names.
narwhals interprets df[0] as row selection, not column selection, causing
AttributeError: 'DataFrame' object has no attribute 'dtype'.
"""
import numpy as np
import pandas as pd
df = pd.DataFrame(np.random.randn(3, 3))
# Must not raise
result = deterministic_dumps(df, hash_type="sha256")
assert isinstance(result, bytes)
@pytest.mark.skipif(
not HAS_PANDAS or not HAS_NUMPY,
reason="pandas and numpy are required",
)
def test_dataframe_same_data_same_hash() -> None:
"""Two DataFrames with identical data must produce the same hash."""
import numpy as np
import pandas as pd
data = np.arange(9, dtype=float).reshape(3, 3)
df1 = pd.DataFrame(data)
df2 = pd.DataFrame(data.copy())
assert deterministic_dumps(df1, hash_type="sha256") == deterministic_dumps(
df2, hash_type="sha256"
)
def test_bytearray_encodes_to_deterministic_bytes() -> None:
"""A bytearray encodes to deterministic bytes that vary by content."""
a = common_container_to_bytes(bytearray(b"abc"))
assert type(a) is bytes
assert a == common_container_to_bytes(bytearray(b"abc"))
assert a != common_container_to_bytes(bytearray(b"xyz"))
def test_bytearray_does_not_collide_with_equal_bytes() -> None:
"""bytearray and bytes of equal content sign under distinct labels."""
assert common_container_to_bytes(bytearray(b"abc")) != (
common_container_to_bytes(b"abc")
)
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars required"
)
def test_polars_numeric_hash_depends_on_values() -> None:
import polars as pl
frame = pl.DataFrame({"x": [1, 2], "y": [3, 4]})
encoded = data_to_buffer(frame)
assert encoded == data_to_buffer(frame.rename({"x": "renamed"}))
assert encoded != data_to_buffer(pl.DataFrame({"x": [1, 2], "y": [3, 5]}))
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars required"
)
@pytest.mark.parametrize("as_frame", [False, True])
def test_polars_non_numeric_state_uses_fallback(as_frame: bool) -> None:
import polars as pl
series = pl.Series("label", ["a", "b"])
data = (
series.to_frame().with_columns(count=pl.Series([1, 2]))
if as_frame
else series
)
value = {"data": data}
assert attempt_signed_bytes(value, "state") is value
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars required"
)
@pytest.mark.parametrize("as_frame", [False, True])
def test_polars_non_numeric_pickle_fallback_preserves_values(
as_frame: bool,
) -> None:
import polars as pl
series = pl.Series("label", ["a", "b"])
value = series.to_frame() if as_frame else series
assert pickle.loads(deterministic_dumps(value, "sha256")).equals(value)
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars required"
)
@pytest.mark.parametrize("as_frame", [False, True])
@pytest.mark.parametrize("in_container", [False, True])
def test_polars_object_hash_uses_fallback(
as_frame: bool, in_container: bool
) -> None:
import polars as pl
series = pl.Series("objects", [object()], dtype=pl.Object)
value: Any = series.to_frame() if as_frame else series
value = {"data": value} if in_container else value
assert attempt_signed_bytes(value, "state") is value