A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(dataframe): preserve empty rows in Polars explode (#11158)
Shahmir Varqhab0de30c62d71
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex d71d489a19..82ae19dfa5 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -10,7 +10,9 @@ import narwhals.stable.v2 as nwfrom narwhals.stable.v2 import colfrom narwhals.typing import IntoLazyFrame+from marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins.ui._impl.dataframes.transforms.print_code import (+ POLARS_EXPLICIT_EXPLODE_MIN_VERSION,python_print_ibis,python_print_pandas,python_print_polars,@@ -502,6 +504,21 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):def handle_explode_columns(df: DataFrame, transform: ExplodeColumnsTransform) -> DataFrame:+ native_df = df.to_native()+ if nw.dependencies.is_polars_lazyframe(native_df):+ # Polars 2 drops empty-list rows unless empty_as_null=True.+ # Narwhals >=2.23 sets these options, available since Polars 1.36.+ # Set them here too so older Narwhals preserves the same rows.+ if DependencyManager.polars.has_at_version(+ min_version=POLARS_EXPLICIT_EXPLODE_MIN_VERSION, quiet=True+ ):+ return nw.from_native(+ native_df.explode(+ transform.column_ids,+ empty_as_null=True,+ keep_nulls=True,+ )+ )return df.explode(transform.column_ids)@staticmethoddiff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.pyindex 56bc587832..ba74d9ec94 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py@@ -11,6 +11,8 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import ()from marimo._utils.assert_never import assert_never+POLARS_EXPLICIT_EXPLODE_MIN_VERSION = "1.36.0"+if TYPE_CHECKING:from collections.abc import Callable@@ -491,7 +493,19 @@ def python_print_polars(elif transform.type == TransformType.EXPLODE_COLUMNS:column_ids = transform.column_ids- return f"{df_name}.explode({_list_of_strings(column_ids)})"+ explode_columns = _list_of_strings(column_ids)+ explode_min_version = tuple(+ map(int, POLARS_EXPLICIT_EXPLODE_MIN_VERSION.split(".")[:2])+ )+ return "\n".join(+ [+ df_name,+ f'if tuple(map(int, pl.__version__.split(".")[:2])) >= {explode_min_version}:',+ f" {df_name} = {df_name}.explode({explode_columns}, empty_as_null=True, keep_nulls=True)",+ "else:",+ f" {df_name} = {df_name}.explode({explode_columns})",+ ]+ )elif transform.type == TransformType.EXPAND_DICT:column_id = _as_literal(transform.column_id)diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex 801de80473..020419c19c 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -1126,6 +1126,66 @@ def test_polars_temporal_conversion_code_matches_runtime()+ not DependencyManager.polars.has(), reason="polars not installed"+)[email protected]("lazy", [False, True])[email protected]("multiple_columns", [False, True])[email protected]("empty_frame", [False, True])+def test_polars_explode_code_preserves_empty_and_null_rows(+ lazy: bool, multiple_columns: bool, empty_frame: bool+) -> None:+ import polars as pl+ import polars.testing as pl_testing++ column = 'list "values"'+ original = pl.DataFrame(+ {+ column: [[1, 2], [], None],+ "other": [[3, 4], [], None],+ "id": [0, 1, 2],+ }+ )+ columns = [column, "other"] if multiple_columns else [column]+ expected = pl.DataFrame(+ {+ column: [1, 2, None, None],+ "other": (+ [3, 4, None, None]+ if multiple_columns+ else [[3, 4], [3, 4], [], None]+ ),+ "id": [0, 0, 1, 2],+ }+ )+ if empty_frame:+ original = original.head(0)+ expected = expected.head(0)+ source = original.lazy() if lazy else original+ transform = ExplodeColumnsTransform(+ type=TransformType.EXPLODE_COLUMNS, column_ids=columns+ )+ runtime = (+ NarwhalsTransformHandler.handle_explode_columns(+ nw.from_native(source).lazy(), transform+ )+ .collect()+ .to_native()+ )+ namespace = {"pl": pl, "df": source}+ exec(+ python_print_transforms(+ "df", original.columns, [transform], python_print_polars+ ),+ namespace,+ )+ generated = namespace["df_next"]+ if lazy:+ generated = generated.collect()+ pl_testing.assert_frame_equal(runtime, expected)+ pl_testing.assert_frame_equal(generated, expected)++@pytest.mark.skipif(not DependencyManager.polars.has(), reason="polars not installed")