A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: align expand-dict transform behavior across dataframe backends (#10110)
Kiran Gadhave780322eb94cb
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex 08b11d8ee2..b18347ad1c 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -503,23 +503,34 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):# Keep pandas handling fully pandas-native so mixed/object columns in# unrelated fields do not trigger Arrow coercion errors.if nw.dependencies.is_pandas_dataframe(native_df):- import math-import pandas as pdresult_df = native_df.copy()+ columns = list(result_df.columns)+ column_index = columns.index(transform.column_id)++ def is_na_like(value: Any) -> bool:+ # is_scalar guards pd.isna, which returns an array for dicts.+ return pd.api.types.is_scalar(value) and bool(pd.isna(value))def normalise_empty_dict(value: Any) -> Any:- if value is None:- return {}- if isinstance(value, float) and math.isnan(value):- return {}- return value+ return {} if is_na_like(value) else value++ column = result_df.pop(transform.column_id)+ if not all(+ isinstance(value, dict) or is_na_like(value)+ for value in column+ ):+ raise nw.exceptions.InvalidOperationError(+ "Expand dict requires a struct-like column with named "+ f"fields on this backend; got column "+ f"'{transform.column_id}' with dtype {column.dtype!r}."+ )# Keep expansion shallow and replace top-level null/nan entries so# pandas and other backends agree on expand-dict behaviour.expanded = pd.json_normalize(- result_df.pop(transform.column_id).map(normalise_empty_dict), # type: ignore[arg-type]+ column.map(normalise_empty_dict), # type: ignore[arg-type]max_level=0,)duplicate_columns = sorted(@@ -531,7 +542,13 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):f"columns: {duplicate_columns}")expanded.index = result_df.index- return undo(nw.from_native(result_df.join(expanded)))+ field_names = list(expanded.columns)+ new_order = (+ columns[:column_index]+ + field_names+ + columns[column_index + 1 :]+ )+ return undo(nw.from_native(result_df.join(expanded)[new_order]))schema = collected_df.collect_schema()dtype = schema.get(transform.column_id)diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.pyindex c2461e56c3..872f551b35 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py@@ -223,9 +223,10 @@ def python_print_pandas(elif transform.type == TransformType.EXPAND_DICT:column_id = _as_literal(transform.column_id)return (- f"{df_name}.join("- f"pd.json_normalize({df_name}.pop({column_id}).map(lambda value: {{}} if value is None or (isinstance(value, float) and value != value) else value), max_level=0).set_axis({df_name}.index, axis=0)"- f")"+ f"{df_name}\n"+ f"_dict_index = {df_name}.columns.get_loc({column_id})\n"+ f"_expanded = pd.json_normalize({df_name}.pop({column_id}).map(lambda value: {{}} if value is None or (isinstance(value, float) and value != value) else value), max_level=0).set_axis({df_name}.index, axis=0)\n"+ f"{df_name} = pd.concat([{df_name}.iloc[:, :_dict_index], _expanded, {df_name}.iloc[:, _dict_index:]], axis=1)")elif transform.type == TransformType.UNIQUE:diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.pyindex c6a6b21bd5..a47e011c23 100644--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py@@ -1857,6 +1857,7 @@ class TestTransformHandler:create_test_dataframes({"A": [1, 2], "B": [10, 20]},include=[+ "pandas","polars","pyarrow","ibis",@@ -1875,6 +1876,46 @@ class TestTransformHandler:):apply(df, transform)+ @staticmethod+ @pytest.mark.parametrize(+ "df",+ create_test_dataframes(+ {+ "id": [1, 2],+ "d": [{"x": 10, "y": 20}, {"x": 30, "y": 40}],+ "name": ["a", "b"],+ },+ include=["pandas", "polars"],+ ),+ )+ def test_expand_dict_preserves_column_position(+ df: DataFrameType,+ ) -> None:+ transform = ExpandDictTransform(+ type=TransformType.EXPAND_DICT, column_id="d"+ )+ result = collect_df(apply(df, transform))+ assert result.columns == ["id", "x", "y", "name"]++ @staticmethod+ def test_expand_dict_pandas_na_like_values_treated_as_empty() -> None:+ pd = pytest.importorskip("pandas")+ df = pd.DataFrame(+ {+ "d": pd.Series(+ [{"x": 1}, None, pd.NA, pd.NaT, float("nan")],+ dtype=object,+ ),+ "id": [1, 2, 3, 4, 5],+ }+ )+ transform = ExpandDictTransform(+ type=TransformType.EXPAND_DICT, column_id="d"+ )+ result = collect_df(apply(df, transform))+ assert result.columns == ["x", "id"]+ assert result["x"].is_null().sum() == 4+@staticmethod@pytest.mark.parametrize((diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex 50b41eb799..fba1d04420 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -568,7 +568,7 @@ def test_print_code_expand_dict_nested_dict_pandas() -> None:real_result,)- assert list(code_result.columns) == ["other", "a", "nested"]+ assert list(code_result.columns) == ["a", "nested", "other"]@pytest.mark.skipif(