A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: serialize pandas via Arrow IPC instead of deprecated feather (#10244)
Shahmir Varqha462de9734b1c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex 9555ff12c8..87aa962be6 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -37,6 +37,17 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()+def _dataframe_to_arrow_ipc(df: pd.DataFrame) -> bytes:+ """Serialize a pandas DataFrame to Arrow IPC bytes."""+ import pyarrow as pa++ out = io.BytesIO()+ table = pa.Table.from_pandas(df)+ with pa.ipc.new_file(out, table.schema) as writer:+ writer.write_table(table)+ return out.getvalue()++def _trivial_range_index(index: pd.Index) -> bool:import pandas as pd@@ -328,26 +339,21 @@ class PandasTableManagerFactory(TableManagerFactory):return pd.api.types.infer_dtype(self._original_data[column])def to_arrow_ipc(self) -> bytes:- out = io.BytesIO()+ import pyarrow as pa+try:- self._original_data.to_feather(- out, compression="uncompressed"- )+ return _dataframe_to_arrow_ipc(self._original_data)except Exception:# Fall back: convert extension-type columns that# PyArrow cannot handle (e.g. pint-pandas) to plain- # values so the feather write can succeed.- import pyarrow as pa-- out = io.BytesIO()+ # values so the IPC write can succeed.df = self._original_data.copy()for col in df.columns:try:pa.Array.from_pandas(df[col])except Exception:df[col] = df[col].astype(object).infer_objects()- df.to_feather(out, compression="uncompressed")- return out.getvalue()+ return _dataframe_to_arrow_ipc(df)def apply_formatting(self, format_mapping: FormatMapping | Nonediff --git a/marimo/_save/stubs/lazy_stub.py b/marimo/_save/stubs/lazy_stub.pyindex 6d0fdd4046..0305090f5b 100644--- a/marimo/_save/stubs/lazy_stub.py+++ b/marimo/_save/stubs/lazy_stub.py@@ -234,27 +234,43 @@ def _npy_dump(obj: Any) -> bytes:return buf.getvalue()+def _pandas_to_arrow_ipc(df: Any) -> bytes:+ # Serialize via Arrow IPC. Prefer LZ4 when available so on-disk cache+ # blobs stay compact.+ import pyarrow as pa++ buf = io.BytesIO()+ table = pa.Table.from_pandas(df)+ options = pa.ipc.IpcWriteOptions(+ compression="lz4" if pa.Codec.is_available("lz4") else None+ )+ with pa.ipc.new_file(buf, table.schema, options=options) as writer:+ writer.write_table(table)+ return buf.getvalue()++def _arrow_dump(obj: Any) -> bytes:# Duck-type dispatch:# polars DataFrame → write_ipc()- # pandas DataFrame → to_feather()+ # pandas DataFrame → Arrow IPC via pyarrow.ipc# Series (either) → to_frame() first, then the appropriate DataFrame method# Fall back to pickle when pyarrow is absent so the cache write never fails.if not DependencyManager.pyarrow.has():return pickle.dumps(obj)- buf = io.BytesIO()if hasattr(obj, "write_ipc"): # polars DataFrame+ buf = io.BytesIO()obj.write_ipc(buf)- elif hasattr(obj, "to_feather"): # pandas DataFrame- obj.to_feather(buf)- else:- # Series — promote to single-column DataFrame, then detect library- frame = obj.to_frame()- if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame- frame.write_ipc(buf)- else: # pandas Series → pandas DataFrame- frame.to_feather(buf)- return buf.getvalue()+ return buf.getvalue()+ if hasattr(obj, "to_feather"): # pandas DataFrame+ return _pandas_to_arrow_ipc(obj)+ # Series — promote to single-column DataFrame, then detect library+ frame = obj.to_frame()+ if hasattr(frame, "write_ipc"): # polars Series → polars DataFrame+ buf = io.BytesIO()+ frame.write_ipc(buf)+ return buf.getvalue()+ # pandas Series → pandas DataFrame+ return _pandas_to_arrow_ipc(frame)def _pt_dump(obj: Any) -> bytes:diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.pyindex 51c2200b5a..540f0d5dd4 100644--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py@@ -2413,28 +2413,30 @@ class TestPandasTableManager(unittest.TestCase):that PyArrow cannot convert (e.g. pint-pandas)."""import pyarrow as pa+ from marimo._plugins.ui._impl.tables import pandas_table as pt+df = pd.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]})manager = self.factory.create()(df)- # Make the first to_feather call raise, simulating an- # unsupported extension dtype.- original_to_feather = pd.DataFrame.to_feather+ # Make the first IPC write raise, simulating an unsupported+ # extension dtype.+ original_write = pt._dataframe_to_arrow_ipccall_count = 0- def patched_to_feather(self_df, *args: Any, **kwargs: Any):+ def patched_write(frame: pd.DataFrame) -> bytes:nonlocal call_countcall_count += 1if call_count == 1:raise pa.lib.ArrowTypeError("unsupported extension type")- return original_to_feather(self_df, *args, **kwargs)+ return original_write(frame)- with patch.object(pd.DataFrame, "to_feather", patched_to_feather):+ with patch.object(pt, "_dataframe_to_arrow_ipc", patched_write):result = manager.to_arrow_ipc()assert isinstance(result, bytes)assert len(result) > 0- # Verify the result is valid IPC/feather data by reading it back+ # Verify the result is valid Arrow IPC data by reading it backbuf = pa.BufferReader(result)table = pa.ipc.open_file(buf).read_all()assert table.num_rows == 3diff --git a/tests/_save/stubs/test_arrow_codecs.py b/tests/_save/stubs/test_arrow_codecs.pynew file mode 100644index 0000000000..2802fe232b--- /dev/null+++ b/tests/_save/stubs/test_arrow_codecs.py@@ -0,0 +1,52 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import warnings++import pytest++from marimo._dependencies.dependencies import DependencyManager+from marimo._save.stubs.lazy_stub import BLOB_DESERIALIZERS, BLOB_SERIALIZERS++HAS_PANDAS_ARROW = (+ DependencyManager.pandas.has() and DependencyManager.pyarrow.has()+)+++def _has_feather_deprecation_warning(+ caught: list[warnings.WarningMessage],+) -> bool:+ return any(+ issubclass(w.category, FutureWarning)+ and "write_feather" in str(w.message)+ for w in caught+ )++[email protected](not HAS_PANDAS_ARROW, reason="pandas and pyarrow required")+def test_pandas_dataframe_arrow_round_trip_without_feather_warning() -> None:+ import pandas as pd++ df = pd.DataFrame({"x": [1, 2], "y": [3.0, 4.0]})+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ data = BLOB_SERIALIZERS["arrow"](df)+ assert not _has_feather_deprecation_warning(caught)++ restored = BLOB_DESERIALIZERS[".arrow"](data, "pandas.DataFrame")+ pd.testing.assert_frame_equal(restored, df)++[email protected](not HAS_PANDAS_ARROW, reason="pandas and pyarrow required")+def test_pandas_series_arrow_round_trip_without_feather_warning() -> None:+ import pandas as pd++ series = pd.Series([10, 20, 30], name="vals")+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ data = BLOB_SERIALIZERS["arrow"](series)+ assert not _has_feather_deprecation_warning(caught)++ restored = BLOB_DESERIALIZERS[".arrow"](data, "pandas.Series")+ assert isinstance(restored, pd.Series)+ pd.testing.assert_series_equal(restored, series)