Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: stringify pint-pandas values in table JSON output (#9951)

Shahmir Varqha9658cdb9650c
diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py
index d351eae5fe..363bb2a426 100644
--- a/marimo/_plugins/ui/_impl/tables/pandas_table.py
+++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import functools
import io
+import json
from functools import cached_property
from typing import TYPE_CHECKING, Any
@@ -11,6 +12,7 @@ import narwhals.stable.v2 as nw
from marimo import _loggers
from marimo._data.models import ExternalDataType
from marimo._dependencies.dependencies import DependencyManager
+from marimo._messaging.msgspec_encoder import enc_hook
from marimo._output.data.data import sanitize_json_bigint
from marimo._plugins.ui._impl.tables.format import (
FormatMapping,
@@ -73,6 +75,28 @@ def _resolve_index_column_conflicts(df: pd.DataFrame) -> pd.DataFrame:
return df
+def _extension_column_needs_stringify(series: pd.Series[Any]) -> bool:
+ """Whether an extension-array column should be cast to str for JSON."""
+ from pandas.api.types import is_extension_array_dtype
+
+ try:
+ if not is_extension_array_dtype(series.dtype):
+ return False
+
+ notna = series.notna()
+ if not notna.any():
+ return False
+
+ # Position-based sample: avoids dropna() copies and .at on
+ # duplicate labels (which can return a Series instead of a scalar).
+ sample = series.iat[int(notna.to_numpy().argmax())]
+ serialized = json.loads(json.dumps(sample, default=enc_hook))
+ return not isinstance(serialized, (str, int, float, bool, type(None)))
+ except Exception:
+ # Conservative fallback: stringify if sampling or serialization fails.
+ return True
+
+
def _maybe_convert_geopandas_to_pandas(data: pd.DataFrame) -> pd.DataFrame:
# Convert to pandas dataframe since geopandas will fail on
# certain operations (like to_json(orient="records"))
@@ -216,6 +240,11 @@ class PandasTableManagerFactory(TableManagerFactory):
# We want to preserve the original display
if is_complex_dtype(dtype):
result[col] = result[col].apply(str)
+ if _extension_column_needs_stringify(result[col]):
+ # Extension arrays with rich Python values (e.g.
+ # pint-pandas) serialize to nested dicts via
+ # to_dict; stringify to preserve display.
+ result[col] = result[col].astype(str)
if is_timedelta64_dtype(
dtype
) or is_timedelta64_ns_dtype(dtype):
@@ -226,9 +255,8 @@ class PandasTableManagerFactory(TableManagerFactory):
inferred_dtype = self._infer_dtype(col)
if inferred_dtype == "date":
result[col] = result[col].apply(str)
-
- # Cast bytes to string to avoid overflow error
- if self._infer_dtype(col) == "bytes":
+ elif inferred_dtype == "bytes":
+ # Cast bytes to string to avoid overflow error
result[col] = result[col].apply(str)
except Exception as e:
diff --git a/marimo/_smoke_tests/tables/extension_array_types.py b/marimo/_smoke_tests/tables/extension_array_types.py
new file mode 100644
index 0000000000..040361ef7d
--- /dev/null
+++ b/marimo/_smoke_tests/tables/extension_array_types.py
@@ -0,0 +1,181 @@
+# /// script
+# requires-python = ">=3.11"
+# dependencies = [
+# "marimo",
+# "pandas",
+# "pint-pandas==0.8.0",
+# "awkward-pandas",
+# "awkward",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.23.10"
+app = marimo.App(width="medium")
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ # Pandas extension-array table display
+
+ Smoke test for rich table rendering of pandas extension dtypes.
+
+ Related: [marimo#9947](https://github.com/marimo-team/marimo/issues/9947)
+
+ Each section shows the default rich table output. Values should be
+ human-readable (e.g. `1.0 meter` for pint-pandas), not nested JSON blobs.
+
+ Use `mo.plain(...)` beside each example to compare against plain HTML.
+ """)
+ return
+
+
+def _():
+ import marimo as mo
+ import pandas as pd
+ from decimal import Decimal
+
+ import awkward as ak
+ import awkward_pandas as akpd
+ import pint_pandas # noqa: F401 — registers pint dtypes with pandas
+
+ return Decimal, ak, akpd, mo, pd
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ ## pint-pandas (float magnitudes)
+
+ Expect readable quantities like `1.0 meter`.
+ """)
+ return
+
+
+def _(mo, pd):
+ pint_float_series = pd.Series([1, 2, 3, 4], dtype="pint[meter]")
+ pint_float_df = pd.DataFrame(
+ {
+ "length": pint_float_series,
+ "width": pd.Series([0.5, 1.0, 1.5, 2.0], dtype="pint[meter]"),
+ }
+ )
+ mo.vstack(
+ [
+ mo.md("**Series**"),
+ mo.hstack(
+ [mo.plain(pint_float_series), pint_float_series], widths="equal"
+ ),
+ mo.md("**DataFrame**"),
+ mo.hstack([mo.plain(pint_float_df), pint_float_df], widths="equal"),
+ ]
+ )
+ return (pint_float_series,)
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ ## pint-pandas (from `Decimal` source values)
+
+ Constructed from `decimal.Decimal` inputs before casting to `pint[meter]`.
+ """)
+ return
+
+
+def _(Decimal, mo, pd):
+ decimal_lengths = pd.Series(
+ [Decimal("1.5"), Decimal("2.25"), Decimal("3.125"), Decimal("4.0"), 1 / 3]
+ )
+ pint_from_decimal = decimal_lengths.astype("pint[meter]")
+ pint_decimal_df = pd.DataFrame(
+ {
+ "raw_decimal": decimal_lengths,
+ "length": pint_from_decimal,
+ }
+ )
+ mo.vstack(
+ [
+ mo.md("**Series (from Decimal → pint)**"),
+ mo.hstack(
+ [mo.plain(pint_from_decimal), pint_from_decimal], widths="equal"
+ ),
+ mo.md("**DataFrame (raw decimal + pint column)**"),
+ mo.hstack(
+ [mo.plain(pint_decimal_df), pint_decimal_df], widths="equal"
+ ),
+ ]
+ )
+ return
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ ## awkward-pandas
+
+ Numeric awkward arrays should stay numeric in the table (not stringified).
+ """)
+ return
+
+
+def _(ak, akpd, mo, pd):
+ awkward_numeric = pd.Series(
+ akpd.AwkwardExtensionArray(ak.Array([1.1, 2.2, 3.3, 4.4]))
+ )
+ awkward_struct = pd.Series(
+ akpd.AwkwardExtensionArray(
+ ak.Array([{"x": 1, "y": 2}, {"x": 3, "y": 4}, {"x": 5, "y": 6}])
+ )
+ )
+ awkward_df = pd.DataFrame(
+ {
+ "values": awkward_numeric,
+ "records": awkward_struct,
+ }
+ )
+ mo.vstack(
+ [
+ mo.md("**Numeric awkward Series**"),
+ mo.hstack(
+ [mo.plain(awkward_numeric), awkward_numeric], widths="equal"
+ ),
+ mo.md("**Struct awkward Series**"),
+ mo.hstack([mo.plain(awkward_struct), awkward_struct], widths="equal"),
+ mo.md("**DataFrame**"),
+ mo.hstack([mo.plain(awkward_df), awkward_df], widths="equal"),
+ ]
+ )
+ return
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ ## Mixed: pint column beside ordinary columns
+ """)
+ return
+
+
+def _(mo, pd, pint_float_series):
+ mixed_df = pd.DataFrame(
+ {
+ "id": [1, 2, 3, 4],
+ "label": ["a", "b", "c", "d"],
+ "length": pint_float_series,
+ "count": pd.Series([10, 20, 30, 40], dtype="Int64"),
+ }
+ )
+ mo.hstack([mo.plain(mixed_df), mixed_df], widths="equal")
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
index e2beb249da..f1fef6907b 100644
--- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py
+++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py
@@ -7,8 +7,9 @@ import re
import unittest
import warnings
from math import isnan, nan
+from types import SimpleNamespace
from typing import Any
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
import narwhals.stable.v2 as nw
import pytest
@@ -20,6 +21,7 @@ from marimo._plugins.ui._impl.table import SortArgs
from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.pandas_table import (
PandasTableManagerFactory,
+ _extension_column_needs_stringify,
)
from marimo._plugins.ui._impl.tables.table_manager import TableManager
from tests.mocks import snapshotter
@@ -2164,13 +2166,73 @@ class TestPandasTableManager(unittest.TestCase):
# MultiIndex should be preserved with original names
assert list(result._original_data.index.names) == ["x", "level"]
+ @pytest.mark.requires("pint_pandas")
+ def test_to_json_str_pint_pandas_series(self) -> None:
+ """pint-pandas quantities display as readable strings in tables."""
+ import pandas as pd
+
+ series = pd.Series([1, 2, 3, 4], dtype="pint[meter]")
+ manager = self.factory.create()(series.to_frame(name="value"))
+ json_str = manager.to_json_str()
+ json_data = json.loads(json_str)
+
+ expected = [{"value": value} for value in series.astype(str)]
+ assert json_data == expected
+
+ def test_extension_column_needs_stringify_skips_json_primitives(
+ self,
+ ) -> None:
+ """Extension columns with JSON-safe scalars should not be stringified."""
+ import pandas as pd
+
+ series = pd.Series([1.1, 2.2, 3.3])
+ with patch(
+ "pandas.api.types.is_extension_array_dtype", return_value=True
+ ):
+ assert not _extension_column_needs_stringify(series)
+
+ def test_extension_column_needs_stringify_for_rich_extension_values(
+ self,
+ ) -> None:
+ """Extension columns with nested JSON values should be stringified."""
+ import pandas as pd
+
+ series = pd.Series([SimpleNamespace(x=1.1)])
+ with patch(
+ "pandas.api.types.is_extension_array_dtype", return_value=True
+ ):
+ assert _extension_column_needs_stringify(series)
+
+ def test_extension_column_needs_stringify_with_duplicate_index(
+ self,
+ ) -> None:
+ """Duplicate labels should not break extension-array sampling."""
+ import pandas as pd
+
+ series = pd.Series([1.1, 2.2], index=[0, 0])
+ with patch(
+ "pandas.api.types.is_extension_array_dtype", return_value=True
+ ):
+ assert not _extension_column_needs_stringify(series)
+
+ def test_to_json_str_keeps_numeric_extension_columns(self) -> None:
+ """Numeric extension-array columns stay numeric in table JSON."""
+ import pandas as pd
+
+ df = pd.DataFrame({"value": [1.1, 2.2, 3.3]})
+ manager = self.factory.create()(df)
+ with patch(
+ "pandas.api.types.is_extension_array_dtype", return_value=True
+ ):
+ json_data = json.loads(manager.to_json_str())
+
+ assert json_data == [{"value": 1.1}, {"value": 2.2}, {"value": 3.3}]
+
def test_to_arrow_ipc_fallback_for_unsupported_extension_dtype(
self,
) -> None:
"""to_arrow_ipc falls back when a column has an extension dtype
that PyArrow cannot convert (e.g. pint-pandas)."""
- from unittest.mock import patch
-
import pyarrow as pa
df = pd.DataFrame({"a": [1, 2, 3], "b": [4.0, 5.0, 6.0]})