Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(table): preserve nulls in pandas JSON stringification (#10517)

Kiran Gadhave33c80ad1917c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index 87aa962be6..204a5d84b0 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -130,21 +130,39 @@ def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:
return df
-def _extension_column_needs_stringify(series: pd.Series[Any]) -> bool:
- """Whether an extension-array column should be cast to str for JSON."""
+def _stringify_preserving_nulls(
+ series: pd.Series[Any],
+ notna_mask: pd.Series[bool] | None = None,
+) -> pd.Series[Any]:
+ """Convert values to strings and preserve missing values."""
+ if notna_mask is None:
+ notna_mask = series.notna()
+
+ stringified = series.apply(str)
+ if not notna_mask.all():
+ stringified = stringified.astype(object).where(notna_mask, None)
+ return stringified
+
+
+def _extension_column_needs_stringify(
+ series: pd.Series[Any],
+ notna_mask: pd.Series[bool] | None = None,
+) -> bool:
+ """Whether an extension-array column needs a string cast for JSON."""
from pandas.api.types import is_extension_array_dtype
try:
if not is_extension_array_dtype(series.dtype):
return False
- notna = series.notna()
- if not notna.any():
+ if notna_mask is None:
+ notna_mask = series.notna()
+ if not notna_mask.any():
return False
# Position-based sample: avoids dropna() copies and .at on
# duplicate labels (which can return a Series instead of a scalar).
- sample = series.iat[int(notna.to_numpy().argmax())]
+ sample = series.iat[int(notna_mask.to_numpy().argmax())]
serialized = json.loads(json.dumps(sample, default=enc_hook))
return not isinstance(serialized, (str, int, float, bool, type(None)))
except Exception:
@@ -281,6 +299,7 @@ class PandasTableManagerFactory(TableManagerFactory):
from pandas.api.types import (
is_complex_dtype,
+ is_extension_array_dtype,
is_object_dtype,
is_timedelta64_dtype,
is_timedelta64_ns_dtype,
@@ -290,29 +309,43 @@ class PandasTableManagerFactory(TableManagerFactory):
result = _data.copy() # to avoid SettingWithCopyWarning
try:
for col in result.columns:
- dtype = result[col].dtype
- # Complex dtypes are converted to {'imag': num, 'real': num} by default
- # We want to preserve the original display
+ series = result[col]
+ dtype = series.dtype
+ # Complex dtypes become {'imag': num, 'real': num} by default.
+ # Preserve their display values instead.
if is_complex_dtype(dtype):
- result[col] = result[col].apply(str)
- if _extension_column_needs_stringify(result[col]):
- # Extension arrays with rich Python values (e.g.
- # pint-pandas) serialize to nested dicts via
- # to_dict; stringify to preserve display.
- result[col] = result[col].astype(str)
+ result[col] = _stringify_preserving_nulls(series)
+
+ notna_mask = (
+ series.notna()
+ if is_extension_array_dtype(dtype)
+ else None
+ )
+ if _extension_column_needs_stringify(
+ series, notna_mask
+ ):
+ # Extension arrays with rich Python values serialize to nested
+ # dictionaries through to_dict. Preserve their display values.
+ result[col] = _stringify_preserving_nulls(
+ series, notna_mask
+ )
+
if is_timedelta64_dtype(
dtype
) or is_timedelta64_ns_dtype(dtype):
- result[col] = result[col].apply(str)
+ result[col] = _stringify_preserving_nulls(series)
if is_object_dtype(dtype):
- # Check if column contains date objects (not datetime), and convert them to string
- # Typically, this will change to YYYY-MM-DD format
+ # Convert date objects to the YYYY-MM-DD display form.
inferred_dtype = self._infer_dtype(col)
if inferred_dtype == "date":
- result[col] = result[col].apply(str)
+ result[col] = _stringify_preserving_nulls(
+ series
+ )
elif inferred_dtype == "bytes":
- # Cast bytes to string to avoid overflow error
- result[col] = result[col].apply(str)
+ # Convert bytes to strings to avoid an overflow error.
+ result[col] = _stringify_preserving_nulls(
+ series
+ )
except Exception as e:
LOGGER.error(
diff --git a/marimo/_smoke_tests/tables/pandas_null_stringify.py b/marimo/_smoke_tests/tables/pandas_null_stringify.py
new file mode 100644
index 0000000000..51a786809e
--- /dev/null
+++ b/marimo/_smoke_tests/tables/pandas_null_stringify.py
@@ -0,0 +1,67 @@
+import marimo
+
+__generated_with = "0.23.16"
+app = marimo.App(width="medium")
+
+
+def _():
+ import json
+ from datetime import date
+
+ import pandas as pd
+
+ from marimo._plugins.ui._impl.tables.pandas_table import (
+ PandasTableManagerFactory,
+ )
+
+ return PandasTableManagerFactory, date, json, pd
+
+
+def _(date, pd):
+ dataframe = pd.DataFrame(
+ {
+ "complex": [
+ 1 + 2j,
+ complex(float("nan"), float("nan")),
+ ],
+ "timedelta": [pd.Timedelta(days=1), pd.NaT],
+ "date": [date(2020, 1, 1), None],
+ "bytes": [b"ab", None],
+ }
+ )
+ dataframe["extension"] = pd.Series(
+ [(1.0,), None],
+ dtype="category",
+ )
+ dataframe
+ return (dataframe,)
+
+
+def _(PandasTableManagerFactory, dataframe, json):
+ manager = PandasTableManagerFactory.create()(dataframe)
+ json_data = json.loads(manager.to_json_str())
+ assert json_data == [
+ {
+ "complex": "(1+2j)",
+ "timedelta": "1 days 00:00:00",
+ "date": "2020-01-01",
+ "bytes": "b'ab'",
+ "extension": "(1.0,)",
+ },
+ {
+ "complex": None,
+ "timedelta": None,
+ "date": None,
+ "bytes": None,
+ "extension": None,
+ },
+ ]
+ json_data
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index b183021c8a..dad7e7e902 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -22,6 +22,7 @@ from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.pandas_table import (
PandasTableManagerFactory,
_extension_column_needs_stringify,
+ _stringify_preserving_nulls,
)
from marimo._plugins.ui._impl.tables.table_manager import TableManager
from tests.mocks import snapshotter
@@ -362,6 +363,33 @@ class TestPandasTableManager(unittest.TestCase):
json_data = self.factory_create_json_from_df(df)
assert json_data[0]["complex"] == "(1+2j)"
+ def test_to_json_stringified_dtypes_preserve_nulls(self) -> None:
+ df = pd.DataFrame(
+ {
+ "complex": [1 + 2j, complex(nan, nan)],
+ "timedelta": [pd.Timedelta(days=1), pd.NaT],
+ "date": [datetime.date(2020, 1, 1), None],
+ "bytes": [b"ab", None],
+ }
+ )
+
+ json_data = self.factory_create_json_from_df(df)
+
+ assert json_data == [
+ {
+ "complex": "(1+2j)",
+ "timedelta": "1 days 00:00:00",
+ "date": "2020-01-01",
+ "bytes": "b'ab'",
+ },
+ {
+ "complex": None,
+ "timedelta": None,
+ "date": None,
+ "bytes": None,
+ },
+ ]
+
@pytest.mark.skipif(
not DependencyManager.numpy.has(),
reason="numpy not installed",
@@ -2388,6 +2416,30 @@ class TestPandasTableManager(unittest.TestCase):
):
assert not _extension_column_needs_stringify(series)
+ def test_stringify_preserving_nulls(self) -> None:
+ series = pd.Series(["value", None], dtype="string")
+
+ result = _stringify_preserving_nulls(series)
+
+ assert result.dtype == object
+ assert result.tolist() == ["value", None]
+
+ def test_stringify_preserving_nulls_skips_mask_for_complete_series(
+ self,
+ ) -> None:
+ series = Mock()
+ notna_mask = Mock()
+ stringified = Mock()
+ series.apply.return_value = stringified
+ notna_mask.all.return_value = True
+
+ result = _stringify_preserving_nulls(series, notna_mask)
+
+ assert result is stringified
+ series.notna.assert_not_called()
+ stringified.astype.assert_not_called()
+ stringified.where.assert_not_called()
+
def test_extension_column_needs_stringify_for_rich_extension_values(
self,
) -> None:
@@ -2425,6 +2477,14 @@ class TestPandasTableManager(unittest.TestCase):
assert json_data == [{"value": 1.1}, {"value": 2.2}, {"value": 3.3}]
+ def test_to_json_str_extension_column_preserves_null(self) -> None:
+ series = pd.Series([(1.0,), None], dtype="category")
+ manager = self.factory.create()(series.to_frame(name="value"))
+
+ json_data = json.loads(manager.to_json_str())
+
+ assert json_data == [{"value": "(1.0,)"}, {"value": None}]
+
def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(
self,
) -> None: