A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(table): preserve nulls in pandas JSON stringification (#10517)
Kiran Gadhave33c80ad1917c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex 87aa962be6..204a5d84b0 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -130,21 +130,39 @@ def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:return df-def _extension_column_needs_stringify(series: pd.Series[Any]) -> bool:- """Whether an extension-array column should be cast to str for JSON."""+def _stringify_preserving_nulls(+ series: pd.Series[Any],+ notna_mask: pd.Series[bool] | None = None,+) -> pd.Series[Any]:+ """Convert values to strings and preserve missing values."""+ if notna_mask is None:+ notna_mask = series.notna()++ stringified = series.apply(str)+ if not notna_mask.all():+ stringified = stringified.astype(object).where(notna_mask, None)+ return stringified+++def _extension_column_needs_stringify(+ series: pd.Series[Any],+ notna_mask: pd.Series[bool] | None = None,+) -> bool:+ """Whether an extension-array column needs a string cast for JSON."""from pandas.api.types import is_extension_array_dtypetry:if not is_extension_array_dtype(series.dtype):return False- notna = series.notna()- if not notna.any():+ if notna_mask is None:+ notna_mask = series.notna()+ if not notna_mask.any():return False# Position-based sample: avoids dropna() copies and .at on# duplicate labels (which can return a Series instead of a scalar).- sample = series.iat[int(notna.to_numpy().argmax())]+ sample = series.iat[int(notna_mask.to_numpy().argmax())]serialized = json.loads(json.dumps(sample, default=enc_hook))return not isinstance(serialized, (str, int, float, bool, type(None)))except Exception:@@ -281,6 +299,7 @@ class PandasTableManagerFactory(TableManagerFactory):from pandas.api.types import (is_complex_dtype,+ is_extension_array_dtype,is_object_dtype,is_timedelta64_dtype,is_timedelta64_ns_dtype,@@ -290,29 +309,43 @@ class PandasTableManagerFactory(TableManagerFactory):result = _data.copy() # to avoid SettingWithCopyWarningtry:for col in result.columns:- dtype = result[col].dtype- # Complex dtypes are converted to {'imag': num, 'real': num} by default- # We want to preserve the original display+ series = result[col]+ dtype = series.dtype+ # Complex dtypes become {'imag': num, 'real': num} by default.+ # Preserve their display values instead.if is_complex_dtype(dtype):- result[col] = result[col].apply(str)- if _extension_column_needs_stringify(result[col]):- # Extension arrays with rich Python values (e.g.- # pint-pandas) serialize to nested dicts via- # to_dict; stringify to preserve display.- result[col] = result[col].astype(str)+ result[col] = _stringify_preserving_nulls(series)++ notna_mask = (+ series.notna()+ if is_extension_array_dtype(dtype)+ else None+ )+ if _extension_column_needs_stringify(+ series, notna_mask+ ):+ # Extension arrays with rich Python values serialize to nested+ # dictionaries through to_dict. Preserve their display values.+ result[col] = _stringify_preserving_nulls(+ series, notna_mask+ )+if is_timedelta64_dtype(dtype) or is_timedelta64_ns_dtype(dtype):- result[col] = result[col].apply(str)+ result[col] = _stringify_preserving_nulls(series)if is_object_dtype(dtype):- # Check if column contains date objects (not datetime), and convert them to string- # Typically, this will change to YYYY-MM-DD format+ # Convert date objects to the YYYY-MM-DD display form.inferred_dtype = self._infer_dtype(col)if inferred_dtype == "date":- result[col] = result[col].apply(str)+ result[col] = _stringify_preserving_nulls(+ series+ )elif inferred_dtype == "bytes":- # Cast bytes to string to avoid overflow error- result[col] = result[col].apply(str)+ # Convert bytes to strings to avoid an overflow error.+ result[col] = _stringify_preserving_nulls(+ series+ )except Exception as e:LOGGER.error(diff --git a/marimo/_smoke_tests/tables/pandas_null_stringify.py b/marimo/_smoke_tests/tables/pandas_null_stringify.pynew file mode 100644index 0000000000..51a786809e--- /dev/null+++ b/marimo/_smoke_tests/tables/pandas_null_stringify.py@@ -0,0 +1,67 @@+import marimo++__generated_with = "0.23.16"+app = marimo.App(width="medium")+++def _():+ import json+ from datetime import date++ import pandas as pd++ from marimo._plugins.ui._impl.tables.pandas_table import (+ PandasTableManagerFactory,+ )++ return PandasTableManagerFactory, date, json, pd+++def _(date, pd):+ dataframe = pd.DataFrame(+ {+ "complex": [+ 1 + 2j,+ complex(float("nan"), float("nan")),+ ],+ "timedelta": [pd.Timedelta(days=1), pd.NaT],+ "date": [date(2020, 1, 1), None],+ "bytes": [b"ab", None],+ }+ )+ dataframe["extension"] = pd.Series(+ [(1.0,), None],+ dtype="category",+ )+ dataframe+ return (dataframe,)+++def _(PandasTableManagerFactory, dataframe, json):+ manager = PandasTableManagerFactory.create()(dataframe)+ json_data = json.loads(manager.to_json_str())+ assert json_data == [+ {+ "complex": "(1+2j)",+ "timedelta": "1 days 00:00:00",+ "date": "2020-01-01",+ "bytes": "b'ab'",+ "extension": "(1.0,)",+ },+ {+ "complex": None,+ "timedelta": None,+ "date": None,+ "bytes": None,+ "extension": None,+ },+ ]+ json_data+ return+++if __name__ == "__main__":+ app.run()diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.pyindex b183021c8a..dad7e7e902 100644--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py@@ -22,6 +22,7 @@ from marimo._plugins.ui._impl.tables.format import FormatMappingfrom marimo._plugins.ui._impl.tables.pandas_table import (PandasTableManagerFactory,_extension_column_needs_stringify,+ _stringify_preserving_nulls,)from marimo._plugins.ui._impl.tables.table_manager import TableManagerfrom tests.mocks import snapshotter@@ -362,6 +363,33 @@ class TestPandasTableManager(unittest.TestCase):json_data = self.factory_create_json_from_df(df)assert json_data[0]["complex"] == "(1+2j)"+ def test_to_json_stringified_dtypes_preserve_nulls(self) -> None:+ df = pd.DataFrame(+ {+ "complex": [1 + 2j, complex(nan, nan)],+ "timedelta": [pd.Timedelta(days=1), pd.NaT],+ "date": [datetime.date(2020, 1, 1), None],+ "bytes": [b"ab", None],+ }+ )++ json_data = self.factory_create_json_from_df(df)++ assert json_data == [+ {+ "complex": "(1+2j)",+ "timedelta": "1 days 00:00:00",+ "date": "2020-01-01",+ "bytes": "b'ab'",+ },+ {+ "complex": None,+ "timedelta": None,+ "date": None,+ "bytes": None,+ },+ ]+@pytest.mark.skipif(not DependencyManager.numpy.has(),reason="numpy not installed",@@ -2388,6 +2416,30 @@ class TestPandasTableManager(unittest.TestCase):):assert not _extension_column_needs_stringify(series)+ def test_stringify_preserving_nulls(self) -> None:+ series = pd.Series(["value", None], dtype="string")++ result = _stringify_preserving_nulls(series)++ assert result.dtype == object+ assert result.tolist() == ["value", None]++ def test_stringify_preserving_nulls_skips_mask_for_complete_series(+ self,+ ) -> None:+ series = Mock()+ notna_mask = Mock()+ stringified = Mock()+ series.apply.return_value = stringified+ notna_mask.all.return_value = True++ result = _stringify_preserving_nulls(series, notna_mask)++ assert result is stringified+ series.notna.assert_not_called()+ stringified.astype.assert_not_called()+ stringified.where.assert_not_called()+def test_extension_column_needs_stringify_for_rich_extension_values(self,) -> None:@@ -2425,6 +2477,14 @@ class TestPandasTableManager(unittest.TestCase):assert json_data == [{"value": 1.1}, {"value": 2.2}, {"value": 3.3}]+ def test_to_json_str_extension_column_preserves_null(self) -> None:+ series = pd.Series([(1.0,), None], dtype="category")+ manager = self.factory.create()(series.to_frame(name="value"))++ json_data = json.loads(manager.to_json_str())++ assert json_data == [{"value": "(1.0,)"}, {"value": None}]+def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(self,) -> None: