Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: align expand-dict transform behavior across dataframe backends (#10110)

Kiran Gadhave780322eb94cb
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index 08b11d8ee2..b18347ad1c 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -503,23 +503,34 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
# Keep pandas handling fully pandas-native so mixed/object columns in
# unrelated fields do not trigger Arrow coercion errors.
if nw.dependencies.is_pandas_dataframe(native_df):
- import math
-
import pandas as pd
result_df = native_df.copy()
+ columns = list(result_df.columns)
+ column_index = columns.index(transform.column_id)
+
+ def is_na_like(value: Any) -> bool:
+ # is_scalar guards pd.isna, which returns an array for dicts.
+ return pd.api.types.is_scalar(value) and bool(pd.isna(value))
def normalise_empty_dict(value: Any) -> Any:
- if value is None:
- return {}
- if isinstance(value, float) and math.isnan(value):
- return {}
- return value
+ return {} if is_na_like(value) else value
+
+ column = result_df.pop(transform.column_id)
+ if not all(
+ isinstance(value, dict) or is_na_like(value)
+ for value in column
+ ):
+ raise nw.exceptions.InvalidOperationError(
+ "Expand dict requires a struct-like column with named "
+ f"fields on this backend; got column "
+ f"'{transform.column_id}' with dtype {column.dtype!r}."
+ )
# Keep expansion shallow and replace top-level null/nan entries so
# pandas and other backends agree on expand-dict behaviour.
expanded = pd.json_normalize(
- result_df.pop(transform.column_id).map(normalise_empty_dict), # type: ignore[arg-type]
+ column.map(normalise_empty_dict), # type: ignore[arg-type]
max_level=0,
)
duplicate_columns = sorted(
@@ -531,7 +542,13 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
f"columns: {duplicate_columns}"
)
expanded.index = result_df.index
- return undo(nw.from_native(result_df.join(expanded)))
+ field_names = list(expanded.columns)
+ new_order = (
+ columns[:column_index]
+ + field_names
+ + columns[column_index + 1 :]
+ )
+ return undo(nw.from_native(result_df.join(expanded)[new_order]))
schema = collected_df.collect_schema()
dtype = schema.get(transform.column_id)
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
index c2461e56c3..872f551b35 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
@@ -223,9 +223,10 @@ def python_print_pandas(
elif transform.type == TransformType.EXPAND_DICT:
column_id = _as_literal(transform.column_id)
return (
- f"{df_name}.join("
- f"pd.json_normalize({df_name}.pop({column_id}).map(lambda value: {{}} if value is None or (isinstance(value, float) and value != value) else value), max_level=0).set_axis({df_name}.index, axis=0)"
- f")"
+ f"{df_name}\n"
+ f"_dict_index = {df_name}.columns.get_loc({column_id})\n"
+ f"_expanded = pd.json_normalize({df_name}.pop({column_id}).map(lambda value: {{}} if value is None or (isinstance(value, float) and value != value) else value), max_level=0).set_axis({df_name}.index, axis=0)\n"
+ f"{df_name} = pd.concat([{df_name}.iloc[:, :_dict_index], _expanded, {df_name}.iloc[:, _dict_index:]], axis=1)"
)
elif transform.type == TransformType.UNIQUE:
diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
index c6a6b21bd5..a47e011c23 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
@@ -1857,6 +1857,7 @@ class TestTransformHandler:
create_test_dataframes(
{"A": [1, 2], "B": [10, 20]},
include=[
+ "pandas",
"polars",
"pyarrow",
"ibis",
@@ -1875,6 +1876,46 @@ class TestTransformHandler:
):
apply(df, transform)
+ @staticmethod
+ @pytest.mark.parametrize(
+ "df",
+ create_test_dataframes(
+ {
+ "id": [1, 2],
+ "d": [{"x": 10, "y": 20}, {"x": 30, "y": 40}],
+ "name": ["a", "b"],
+ },
+ include=["pandas", "polars"],
+ ),
+ )
+ def test_expand_dict_preserves_column_position(
+ df: DataFrameType,
+ ) -> None:
+ transform = ExpandDictTransform(
+ type=TransformType.EXPAND_DICT, column_id="d"
+ )
+ result = collect_df(apply(df, transform))
+ assert result.columns == ["id", "x", "y", "name"]
+
+ @staticmethod
+ def test_expand_dict_pandas_na_like_values_treated_as_empty() -> None:
+ pd = pytest.importorskip("pandas")
+ df = pd.DataFrame(
+ {
+ "d": pd.Series(
+ [{"x": 1}, None, pd.NA, pd.NaT, float("nan")],
+ dtype=object,
+ ),
+ "id": [1, 2, 3, 4, 5],
+ }
+ )
+ transform = ExpandDictTransform(
+ type=TransformType.EXPAND_DICT, column_id="d"
+ )
+ result = collect_df(apply(df, transform))
+ assert result.columns == ["x", "id"]
+ assert result["x"].is_null().sum() == 4
+
@staticmethod
@pytest.mark.parametrize(
(
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index 50b41eb799..fba1d04420 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -568,7 +568,7 @@ def test_print_code_expand_dict_nested_dict_pandas() -> None:
real_result,
)
- assert list(code_result.columns) == ["other", "a", "nested"]
+ assert list(code_result.columns) == ["a", "nested", "other"]
@pytest.mark.skipif(