A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: stringify pint-pandas values in table JSON output (#9951)
Shahmir Varqha9658cdb9650c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.pyindex d351eae5fe..363bb2a426 100644--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport functoolsimport io+import jsonfrom functools import cached_propertyfrom typing import TYPE_CHECKING, Any@@ -11,6 +12,7 @@ import narwhals.stable.v2 as nwfrom marimo import _loggersfrom marimo._data.models import ExternalDataTypefrom marimo._dependencies.dependencies import DependencyManager+from marimo._messaging.msgspec_encoder import enc_hookfrom marimo._output.data.data import sanitize_json_bigintfrom marimo._plugins.ui._impl.tables.format import (FormatMapping,@@ -73,6 +75,28 @@ def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:return df+def _extension_column_needs_stringify(series: pd.Series[Any]) -> bool:+ """Whether an extension-array column should be cast to str for JSON."""+ from pandas.api.types import is_extension_array_dtype++ try:+ if not is_extension_array_dtype(series.dtype):+ return False++ notna = series.notna()+ if not notna.any():+ return False++ # Position-based sample: avoids dropna() copies and .at on+ # duplicate labels (which can return a Series instead of a scalar).+ sample = series.iat[int(notna.to_numpy().argmax())]+ serialized = json.loads(json.dumps(sample, default=enc_hook))+ return not isinstance(serialized, (str, int, float, bool, type(None)))+ except Exception:+ # Conservative fallback: stringify if sampling or serialization fails.+ return True++def _maybe_convert_geopandas_to_pandas(data: pd.DataFrame) -> pd.DataFrame:# Convert to pandas dataframe since geopandas will fail on# certain operations (like to_json(orient="records"))@@ -216,6 +240,11 @@ class PandasTableManagerFactory(TableManagerFactory):# We want to preserve the original displayif is_complex_dtype(dtype):result[col] = result[col].apply(str)+ if _extension_column_needs_stringify(result[col]):+ # Extension arrays with rich Python values (e.g.+ # pint-pandas) serialize to nested dicts via+ # to_dict; stringify to preserve display.+ result[col] = result[col].astype(str)if is_timedelta64_dtype(dtype) or is_timedelta64_ns_dtype(dtype):@@ -226,9 +255,8 @@ class PandasTableManagerFactory(TableManagerFactory):inferred_dtype = self._infer_dtype(col)if inferred_dtype == "date":result[col] = result[col].apply(str)-- # Cast bytes to string to avoid overflow error- if self._infer_dtype(col) == "bytes":+ elif inferred_dtype == "bytes":+ # Cast bytes to string to avoid overflow errorresult[col] = result[col].apply(str)except Exception as e:diff --git a/marimo/_smoke_tests/tables/extension_array_types.py b/marimo/_smoke_tests/tables/extension_array_types.pynew file mode 100644index 0000000000..040361ef7d--- /dev/null+++ b/marimo/_smoke_tests/tables/extension_array_types.py@@ -0,0 +1,181 @@+# /// script+# requires-python = ">=3.11"+# dependencies = [+# "marimo",+# "pandas",+# "pint-pandas==0.8.0",+# "awkward-pandas",+# "awkward",+# ]+# ///++import marimo++__generated_with = "0.23.10"+app = marimo.App(width="medium")++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ # Pandas extension-array table display++ Smoke test for rich table rendering of pandas extension dtypes.++ Related: [marimo#9947](https://github.com/marimo-team/marimo/issues/9947)++ Each section shows the default rich table output. Values should be+ human-readable (e.g. `1.0 meter` for pint-pandas), not nested JSON blobs.++ Use `mo.plain(...)` beside each example to compare against plain HTML.+ """)+ return+++def _():+ import marimo as mo+ import pandas as pd+ from decimal import Decimal++ import awkward as ak+ import awkward_pandas as akpd+ import pint_pandas # noqa: F401 — registers pint dtypes with pandas++ return Decimal, ak, akpd, mo, pd++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ ## pint-pandas (float magnitudes)++ Expect readable quantities like `1.0 meter`.+ """)+ return+++def _(mo, pd):+ pint_float_series = pd.Series([1, 2, 3, 4], dtype="pint[meter]")+ pint_float_df = pd.DataFrame(+ {+ "length": pint_float_series,+ "width": pd.Series([0.5, 1.0, 1.5, 2.0], dtype="pint[meter]"),+ }+ )+ mo.vstack(+ [+ mo.md("**Series**"),+ mo.hstack(+ [mo.plain(pint_float_series), pint_float_series], widths="equal"+ ),+ mo.md("**DataFrame**"),+ mo.hstack([mo.plain(pint_float_df), pint_float_df], widths="equal"),+ ]+ )+ return (pint_float_series,)++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ ## pint-pandas (from `Decimal` source values)++ Constructed from `decimal.Decimal` inputs before casting to `pint[meter]`.+ """)+ return+++def _(Decimal, mo, pd):+ decimal_lengths = pd.Series(+ [Decimal("1.5"), Decimal("2.25"), Decimal("3.125"), Decimal("4.0"), 1 / 3]+ )+ pint_from_decimal = decimal_lengths.astype("pint[meter]")+ pint_decimal_df = pd.DataFrame(+ {+ "raw_decimal": decimal_lengths,+ "length": pint_from_decimal,+ }+ )+ mo.vstack(+ [+ mo.md("**Series (from Decimal → pint)**"),+ mo.hstack(+ [mo.plain(pint_from_decimal), pint_from_decimal], widths="equal"+ ),+ mo.md("**DataFrame (raw decimal + pint column)**"),+ mo.hstack(+ [mo.plain(pint_decimal_df), pint_decimal_df], widths="equal"+ ),+ ]+ )+ return++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ ## awkward-pandas++ Numeric awkward arrays should stay numeric in the table (not stringified).+ """)+ return+++def _(ak, akpd, mo, pd):+ awkward_numeric = pd.Series(+ akpd.AwkwardExtensionArray(ak.Array([1.1, 2.2, 3.3, 4.4]))+ )+ awkward_struct = pd.Series(+ akpd.AwkwardExtensionArray(+ ak.Array([{"x": 1, "y": 2}, {"x": 3, "y": 4}, {"x": 5, "y": 6}])+ )+ )+ awkward_df = pd.DataFrame(+ {+ "values": awkward_numeric,+ "records": awkward_struct,+ }+ )+ mo.vstack(+ [+ mo.md("**Numeric awkward Series**"),+ mo.hstack(+ [mo.plain(awkward_numeric), awkward_numeric], widths="equal"+ ),+ mo.md("**Struct awkward Series**"),+ mo.hstack([mo.plain(awkward_struct), awkward_struct], widths="equal"),+ mo.md("**DataFrame**"),+ mo.hstack([mo.plain(awkward_df), awkward_df], widths="equal"),+ ]+ )+ return++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ ## Mixed: pint column beside ordinary columns+ """)+ return+++def _(mo, pd, pint_float_series):+ mixed_df = pd.DataFrame(+ {+ "id": [1, 2, 3, 4],+ "label": ["a", "b", "c", "d"],+ "length": pint_float_series,+ "count": pd.Series([10, 20, 30, 40], dtype="Int64"),+ }+ )+ mo.hstack([mo.plain(mixed_df), mixed_df], widths="equal")+ return+++if __name__ == "__main__":+ app.run()diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.pyindex e2beb249da..f1fef6907b 100644--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py@@ -7,8 +7,9 @@ import reimport unittestimport warningsfrom math import isnan, nan+from types import SimpleNamespacefrom typing import Any-from unittest.mock import Mock+from unittest.mock import Mock, patchimport narwhals.stable.v2 as nwimport pytest@@ -20,6 +21,7 @@ from marimo._plugins.ui._impl.table import SortArgsfrom marimo._plugins.ui._impl.tables.format import FormatMappingfrom marimo._plugins.ui._impl.tables.pandas_table import (PandasTableManagerFactory,+ _extension_column_needs_stringify,)from marimo._plugins.ui._impl.tables.table_manager import TableManagerfrom tests.mocks import snapshotter@@ -2164,13 +2166,73 @@ class TestPandasTableManager(unittest.TestCase):# MultiIndex should be preserved with original namesassert list(result._original_data.index.names) == ["x", "level"]+ @pytest.mark.requires("pint_pandas")+ def test_to_json_str_pint_pandas_series(self) -> None:+ """pint-pandas quantities display as readable strings in tables."""+ import pandas as pd++ series = pd.Series([1, 2, 3, 4], dtype="pint[meter]")+ manager = self.factory.create()(series.to_frame(name="value"))+ json_str = manager.to_json_str()+ json_data = json.loads(json_str)++ expected = [{"value": value} for value in series.astype(str)]+ assert json_data == expected++ def test_extension_column_needs_stringify_skips_json_primitives(+ self,+ ) -> None:+ """Extension columns with JSON-safe scalars should not be stringified."""+ import pandas as pd++ series = pd.Series([1.1, 2.2, 3.3])+ with patch(+ "pandas.api.types.is_extension_array_dtype", return_value=True+ ):+ assert not _extension_column_needs_stringify(series)++ def test_extension_column_needs_stringify_for_rich_extension_values(+ self,+ ) -> None:+ """Extension columns with nested JSON values should be stringified."""+ import pandas as pd++ series = pd.Series([SimpleNamespace(x=1.1)])+ with patch(+ "pandas.api.types.is_extension_array_dtype", return_value=True+ ):+ assert _extension_column_needs_stringify(series)++ def test_extension_column_needs_stringify_with_duplicate_index(+ self,+ ) -> None:+ """Duplicate labels should not break extension-array sampling."""+ import pandas as pd++ series = pd.Series([1.1, 2.2], index=[0, 0])+ with patch(+ "pandas.api.types.is_extension_array_dtype", return_value=True+ ):+ assert not _extension_column_needs_stringify(series)++ def test_to_json_str_keeps_numeric_extension_columns(self) -> None:+ """Numeric extension-array columns stay numeric in table JSON."""+ import pandas as pd++ df = pd.DataFrame({"value": [1.1, 2.2, 3.3]})+ manager = self.factory.create()(df)+ with patch(+ "pandas.api.types.is_extension_array_dtype", return_value=True+ ):+ json_data = json.loads(manager.to_json_str())++ assert json_data == [{"value": 1.1}, {"value": 2.2}, {"value": 3.3}]+def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(self,) -> None:"""to_arrow_ipc falls back when a column has an extension dtypethat PyArrow cannot convert (e.g. pint-pandas)."""- from unittest.mock import patch-import pyarrow as padf = pd.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]})