Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(dataframe): preserve empty rows in Polars explode (#11158)

Shahmir Varqhab0de30c62d71
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index d71d489a19..82ae19dfa5 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -10,7 +10,9 @@ import narwhals.stable.v2 as nw
from narwhals.stable.v2 import col
from narwhals.typing import IntoLazyFrame
+from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.dataframes.transforms.print_code import (
+ POLARS_EXPLICIT_EXPLODE_MIN_VERSION,
python_print_ibis,
python_print_pandas,
python_print_polars,
@@ -502,6 +504,21 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
def handle_explode_columns(
df: DataFrame, transform: ExplodeColumnsTransform
) -> DataFrame:
+ native_df = df.to_native()
+ if nw.dependencies.is_polars_lazyframe(native_df):
+ # Polars 2 drops empty-list rows unless empty_as_null=True.
+ # Narwhals >=2.23 sets these options, available since Polars 1.36.
+ # Set them here too so older Narwhals preserves the same rows.
+ if DependencyManager.polars.has_at_version(
+ min_version=POLARS_EXPLICIT_EXPLODE_MIN_VERSION, quiet=True
+ ):
+ return nw.from_native(
+ native_df.explode(
+ transform.column_ids,
+ empty_as_null=True,
+ keep_nulls=True,
+ )
+ )
return df.explode(transform.column_ids)
@staticmethod
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
index 56bc587832..ba74d9ec94 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
@@ -11,6 +11,8 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
)
from marimo._utils.assert_never import assert_never
+POLARS_EXPLICIT_EXPLODE_MIN_VERSION = "1.36.0"
+
if TYPE_CHECKING:
from collections.abc import Callable
@@ -491,7 +493,19 @@ def python_print_polars(
elif transform.type == TransformType.EXPLODE_COLUMNS:
column_ids = transform.column_ids
- return f"{df_name}.explode({_list_of_strings(column_ids)})"
+ explode_columns = _list_of_strings(column_ids)
+ explode_min_version = tuple(
+ map(int, POLARS_EXPLICIT_EXPLODE_MIN_VERSION.split(".")[:2])
+ )
+ return "\n".join(
+ [
+ df_name,
+ f'if tuple(map(int, pl.__version__.split(".")[:2])) >= {explode_min_version}:',
+ f" {df_name} = {df_name}.explode({explode_columns}, empty_as_null=True, keep_nulls=True)",
+ "else:",
+ f" {df_name} = {df_name}.explode({explode_columns})",
+ ]
+ )
elif transform.type == TransformType.EXPAND_DICT:
column_id = _as_literal(transform.column_id)
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index 801de80473..020419c19c 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -1126,6 +1126,66 @@ def test_polars_temporal_conversion_code_matches_runtime(
)
+ not DependencyManager.polars.has(), reason="polars not installed"
+)
[email protected]("lazy", [False, True])
[email protected]("multiple_columns", [False, True])
[email protected]("empty_frame", [False, True])
+def test_polars_explode_code_preserves_empty_and_null_rows(
+ lazy: bool, multiple_columns: bool, empty_frame: bool
+) -> None:
+ import polars as pl
+ import polars.testing as pl_testing
+
+ column = 'list "values"'
+ original = pl.DataFrame(
+ {
+ column: [[1, 2], [], None],
+ "other": [[3, 4], [], None],
+ "id": [0, 1, 2],
+ }
+ )
+ columns = [column, "other"] if multiple_columns else [column]
+ expected = pl.DataFrame(
+ {
+ column: [1, 2, None, None],
+ "other": (
+ [3, 4, None, None]
+ if multiple_columns
+ else [[3, 4], [3, 4], [], None]
+ ),
+ "id": [0, 0, 1, 2],
+ }
+ )
+ if empty_frame:
+ original = original.head(0)
+ expected = expected.head(0)
+ source = original.lazy() if lazy else original
+ transform = ExplodeColumnsTransform(
+ type=TransformType.EXPLODE_COLUMNS, column_ids=columns
+ )
+ runtime = (
+ NarwhalsTransformHandler.handle_explode_columns(
+ nw.from_native(source).lazy(), transform
+ )
+ .collect()
+ .to_native()
+ )
+ namespace = {"pl": pl, "df": source}
+ exec(
+ python_print_transforms(
+ "df", original.columns, [transform], python_print_polars
+ ),
+ namespace,
+ )
+ generated = namespace["df_next"]
+ if lazy:
+ generated = generated.collect()
+ pl_testing.assert_frame_equal(runtime, expected)
+ pl_testing.assert_frame_equal(generated, expected)
+
+
@pytest.mark.skipif(
not DependencyManager.polars.has(), reason="polars not installed"
)