Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: serialize pandas via Arrow IPC instead of deprecated feather (#10244)

Shahmir Varqha462de9734b1c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index 9555ff12c8..87aa962be6 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -37,6 +37,17 @@ if TYPE_CHECKING:
LOGGER = _loggers.marimo_logger()
+def _dataframe_to_arrow_ipc(df: pd.DataFrame) -> bytes:
+ """Serialize a pandas DataFrame to Arrow IPC bytes."""
+ import pyarrow as pa
+
+ out = io.BytesIO()
+ table = pa.Table.from_pandas(df)
+ with pa.ipc.new_file(out, table.schema) as writer:
+ writer.write_table(table)
+ return out.getvalue()
+
+
def _trivial_range_index(index: pd.Index) -> bool:
import pandas as pd
@@ -328,26 +339,21 @@ class PandasTableManagerFactory(TableManagerFactory):
return pd.api.types.infer_dtype(self._original_data[column])
def to_arrow_ipc(self) -> bytes:
- out = io.BytesIO()
+ import pyarrow as pa
+
try:
- self._original_data.to_feather(
- out, compression="uncompressed"
- )
+ return _dataframe_to_arrow_ipc(self._original_data)
except Exception:
# Fall back: convert extension-type columns that
# PyArrow cannot handle (e.g. pint-pandas) to plain
- # values so the feather write can succeed.
- import pyarrow as pa
-
- out = io.BytesIO()
+ # values so the IPC write can succeed.
df = self._original_data.copy()
for col in df.columns:
try:
pa.Array.from_pandas(df[col])
except Exception:
df[col] = df[col].astype(object).infer_objects()
- df.to_feather(out, compression="uncompressed")
- return out.getvalue()
+ return _dataframe_to_arrow_ipc(df)
def apply_formatting(
self, format_mapping: FormatMapping | None
diff --git a/marimo/_save/stubs/lazy_stub.py b/marimo/_save/stubs/lazy_stub.py
index 6d0fdd4046..0305090f5b 100644
--- a/marimo/_save/stubs/lazy_stub.py
+++ b/marimo/_save/stubs/lazy_stub.py
@@ -234,27 +234,43 @@ def _npy_dump(obj: Any) -> bytes:
return buf.getvalue()
+def _pandas_to_arrow_ipc(df: Any) -> bytes:
+ # Serialize via Arrow IPC. Prefer LZ4 when available so on-disk cache
+ # blobs stay compact.
+ import pyarrow as pa
+
+ buf = io.BytesIO()
+ table = pa.Table.from_pandas(df)
+ options = pa.ipc.IpcWriteOptions(
+ compression="lz4" if pa.Codec.is_available("lz4") else None
+ )
+ with pa.ipc.new_file(buf, table.schema, options=options) as writer:
+ writer.write_table(table)
+ return buf.getvalue()
+
+
def _arrow_dump(obj: Any) -> bytes:
# Duck-type dispatch:
# polars DataFrame → write_ipc()
- # pandas DataFrame → to_feather()
+ # pandas DataFrame → Arrow IPC via pyarrow.ipc
# Series (either) → to_frame() first, then the appropriate DataFrame method
# Fall back to pickle when pyarrow is absent so the cache write never fails.
if not DependencyManager.pyarrow.has():
return pickle.dumps(obj)
- buf = io.BytesIO()
if hasattr(obj, "write_ipc"): # polars DataFrame
+ buf = io.BytesIO()
obj.write_ipc(buf)
- elif hasattr(obj, "to_feather"): # pandas DataFrame
- obj.to_feather(buf)
- else:
- # Series — promote to single-column DataFrame, then detect library
- frame = obj.to_frame()
- if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame
- frame.write_ipc(buf)
- else: # pandas Series → pandas DataFrame
- frame.to_feather(buf)
- return buf.getvalue()
+ return buf.getvalue()
+ if hasattr(obj, "to_feather"): # pandas DataFrame
+ return _pandas_to_arrow_ipc(obj)
+ # Series — promote to single-column DataFrame, then detect library
+ frame = obj.to_frame()
+ if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame
+ buf = io.BytesIO()
+ frame.write_ipc(buf)
+ return buf.getvalue()
+ # pandas Series → pandas DataFrame
+ return _pandas_to_arrow_ipc(frame)
def _pt_dump(obj: Any) -> bytes:
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index 51c2200b5a..540f0d5dd4 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -2413,28 +2413,30 @@ class TestPandasTableManager(unittest.TestCase):
that PyArrow cannot convert (e.g. pint-pandas)."""
import pyarrow as pa
+ from marimo._plugins.ui._impl.tables import pandas_table as pt
+
df = pd.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]})
manager = self.factory.create()(df)
- # Make the first to_feather call raise, simulating an
- # unsupported extension dtype.
- original_to_feather = pd.DataFrame.to_feather
+ # Make the first IPC write raise, simulating an unsupported
+ # extension dtype.
+ original_write = pt._dataframe_to_arrow_ipc
call_count = 0
- def patched_to_feather(self_df, *args: Any, **kwargs: Any):
+ def patched_write(frame: pd.DataFrame) -> bytes:
nonlocal call_count
call_count += 1
if call_count == 1:
raise pa.lib.ArrowTypeError("unsupported extension type")
- return original_to_feather(self_df, *args, **kwargs)
+ return original_write(frame)
- with patch.object(pd.DataFrame, "to_feather", patched_to_feather):
+ with patch.object(pt, "_dataframe_to_arrow_ipc", patched_write):
result = manager.to_arrow_ipc()
assert isinstance(result, bytes)
assert len(result) > 0
- # Verify the result is valid IPC/feather data by reading it back
+ # Verify the result is valid Arrow IPC data by reading it back
buf = pa.BufferReader(result)
table = pa.ipc.open_file(buf).read_all()
assert table.num_rows == 3
diff --git a/tests/_save/stubs/test_arrow_codecs.py b/tests/_save/stubs/test_arrow_codecs.py
new file mode 100644
index 0000000000..2802fe232b
--- /dev/null
+++ b/tests/_save/stubs/test_arrow_codecs.py
@@ -0,0 +1,52 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import warnings
+
+import pytest
+
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._save.stubs.lazy_stub import BLOB_DESERIALIZERS, BLOB_SERIALIZERS
+
+HAS_PANDAS_ARROW = (
+ DependencyManager.pandas.has() and DependencyManager.pyarrow.has()
+)
+
+
+def _has_feather_deprecation_warning(
+ caught: list[warnings.WarningMessage],
+) -> bool:
+ return any(
+ issubclass(w.category, FutureWarning)
+ and "write_feather" in str(w.message)
+ for w in caught
+ )
+
+
[email protected](not HAS_PANDAS_ARROW, reason="pandas and pyarrow required")
+def test_pandas_dataframe_arrow_round_trip_without_feather_warning() -> None:
+ import pandas as pd
+
+ df = pd.DataFrame({"x": [1, 2], "y": [3.0, 4.0]})
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ data = BLOB_SERIALIZERS["arrow"](df)
+ assert not _has_feather_deprecation_warning(caught)
+
+ restored = BLOB_DESERIALIZERS[".arrow"](data, "pandas.DataFrame")
+ pd.testing.assert_frame_equal(restored, df)
+
+
[email protected](not HAS_PANDAS_ARROW, reason="pandas and pyarrow required")
+def test_pandas_series_arrow_round_trip_without_feather_warning() -> None:
+ import pandas as pd
+
+ series = pd.Series([10, 20, 30], name="vals")
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ data = BLOB_SERIALIZERS["arrow"](series)
+ assert not _has_feather_deprecation_warning(caught)
+
+ restored = BLOB_DESERIALIZERS[".arrow"](data, "pandas.Series")
+ assert isinstance(restored, pd.Series)
+ pd.testing.assert_series_equal(restored, series)