Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(cache): hash pyarrow values with non-numeric columns by execution path (#11084)

Péter Gyarmati82b936f910b6
diff --git a/marimo/_runtime/primitives.py b/marimo/_runtime/primitives.py
index 72e98ded98..471fea9658 100644
--- a/marimo/_runtime/primitives.py
+++ b/marimo/_runtime/primitives.py
@@ -76,6 +76,18 @@ def is_data_primitive(value: Any) -> bool:
if is_instance_by_name(value, "torch.Tensor"):
return str(value.device) == "cpu"
+ # Arrow tables and arrays expose `__array__` without a `dtype`. Only
+ # numeric columns convert to an array that can be hashed as bytes.
+ if type(value).__module__.startswith("pyarrow"):
+ import pyarrow as pa
+
+ schema = getattr(value, "schema", None)
+ types = schema.types if schema is not None else [value.type]
+ return all(
+ pa.types.is_integer(dtype) or pa.types.is_floating(dtype)
+ for dtype in types
+ )
+
is_polars = type(value).__module__.startswith("polars.")
# If a numpy like array, ensure that it's not an object array.
if hasattr(value, "dtype") and not is_polars:
diff --git a/tests/_runtime/test_primitives.py b/tests/_runtime/test_primitives.py
index 3fbe6d9a50..e29ff9e042 100644
--- a/tests/_runtime/test_primitives.py
+++ b/tests/_runtime/test_primitives.py
@@ -51,6 +51,37 @@ class TestDataPrimitiveClassification:
"object": False,
}
+ @pytest.mark.skipif(
+ not DependencyManager.pyarrow.has(), reason="pyarrow required"
+ )
+ def test_pyarrow_numeric_classification(self) -> None:
+ import pyarrow as pa
+
+ values = {
+ "integer_array": pa.array([1, None]),
+ "float_chunks": pa.chunked_array([[1.0], [2.0]]),
+ "numeric_table": pa.table({"a": [1], "b": [1.5]}),
+ "numeric_batch": pa.RecordBatch.from_pydict({"a": [1]}),
+ "string_array": pa.array(["a"]),
+ "string_table": pa.table({"a": [1], "b": ["x"]}),
+ "dictionary_table": pa.table(
+ {"a": pa.array(["x"]).dictionary_encode()}
+ ),
+ "boolean_array": pa.array([True, None]),
+ }
+ assert {
+ name: is_data_primitive(value) for name, value in values.items()
+ } == {
+ "integer_array": True,
+ "float_chunks": True,
+ "numeric_table": True,
+ "numeric_batch": True,
+ "string_array": False,
+ "string_table": False,
+ "dictionary_table": False,
+ "boolean_array": False,
+ }
+
class TestWrappedFunctionHandling:
"""Test handling of wrapped functions (decorators) in is_pure_function."""
diff --git a/tests/_save/test_hash.py b/tests/_save/test_hash.py
index 4bf9cb90f0..69e62035a0 100644
--- a/tests/_save/test_hash.py
+++ b/tests/_save/test_hash.py
@@ -1230,6 +1230,32 @@ class TestDataHash:
assert _A == 28
return (two,)
+ @staticmethod
+ @pytest.mark.skipif(
+ not DependencyManager.has("pyarrow"),
+ reason="optional dependencies not installed",
+ )
+ def test_pyarrow_string_table(app) -> None:
+ @app.cell
+ def load() -> tuple[Any]:
+ import pyarrow as pa
+
+ from marimo._save.save import persistent_cache
+ from tests._save.loaders.mocks import MockLoader
+
+ return MockLoader, persistent_cache, pa
+
+ @app.cell
+ def two(MockLoader, persistent_cache, pa) -> tuple[int]:
+ _a = pa.table({"A": [2, 8, 18], "B": ["a", "b", "c"]})
+
+ with persistent_cache(name="two", _loader=MockLoader()) as _cache:
+ _A = _a.column("A").to_pylist()
+
+ assert _cache._cache.cache_type == "ContextExecutionPath"
+ assert _A == [2, 8, 18]
+ return (two,)
+
class TestCustomHash:
@staticmethod