Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: preserve null handling in generated filters (#11036)

Kiran Gadhave719f967b9dff
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
index d452437e36..ced183dc58 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py
@@ -49,13 +49,13 @@ def python_print_pandas(
elif operator in [">", ">=", "<", "<="]:
result = f"{col} {operator} {_as_literal(value)}"
elif operator == "contains":
- result = f"{col}.str.contains({_as_literal(value)}, regex=False)"
+ result = f'{col}.fillna("").str.contains({_as_literal(value)}, regex=False)'
elif operator == "regex":
- result = f"{col}.str.contains({_as_literal(value)}, regex=True)"
+ result = f'{col}.fillna("").str.contains({_as_literal(value)}, regex=True)'
elif operator == "starts_with":
- result = f"{col}.str.startswith({_as_literal(value)})"
+ result = f'{col}.fillna("").str.startswith({_as_literal(value)})'
elif operator == "ends_with":
- result = f"{col}.str.endswith({_as_literal(value)})"
+ result = f'{col}.fillna("").str.endswith({_as_literal(value)})'
elif operator == "in" or operator == "not_in":
expr = f"{col}.isin({_list_of_strings(value)})"
result = expr if operator == "in" else f"~{expr}"
@@ -81,6 +81,8 @@ def python_print_pandas(
if where.negate:
result = f"~({result})"
+ if operator in {"contains", "regex", "starts_with", "ends_with"}:
+ result = f"({result} & {col}.notna())"
return result
@@ -317,13 +319,15 @@ def python_print_polars(
elif operator in [">", ">=", "<", "<="]:
result = f"{col} {operator} {_as_literal(value)}"
elif operator == "contains":
- result = f"{col}.str.contains({_as_literal(value)}, literal=True)"
+ result = f'{col}.fill_null("").str.contains({_as_literal(value)}, literal=True)'
elif operator == "regex":
- result = f"{col}.str.contains({_as_literal(value)}, literal=False)"
+ result = f'{col}.fill_null("").str.contains({_as_literal(value)}, literal=False)'
elif operator == "starts_with":
- result = f"{col}.str.starts_with({_as_literal(value)})"
+ result = (
+ f'{col}.fill_null("").str.starts_with({_as_literal(value)})'
+ )
elif operator == "ends_with":
- result = f"{col}.str.ends_with({_as_literal(value)})"
+ result = f'{col}.fill_null("").str.ends_with({_as_literal(value)})'
elif operator == "in" or operator == "not_in":
expr = f"{col}.is_in({_list_of_strings(value)})"
result = expr if operator == "in" else f"~{expr}"
@@ -349,6 +353,8 @@ def python_print_polars(
if where.negate:
result = f"~({result})"
+ if operator in {"contains", "regex", "starts_with", "ends_with"}:
+ result = f"({result} & {col}.is_not_null())"
return result
@@ -554,13 +560,13 @@ def python_print_ibis(
elif operator in [">", ">=", "<", "<="]:
result = f"({col} {operator} {_as_literal(value)})"
elif operator == "contains":
- result = f"({col}.contains({_as_literal(value)}))"
+ result = f'({col}.fill_null("").contains({_as_literal(value)}))'
elif operator == "regex":
- result = f"({col}.re_search({_as_literal(value)}))"
+ result = f'({col}.fill_null("").re_search({_as_literal(value)}))'
elif operator == "starts_with":
- result = f"({col}.startswith({_as_literal(value)}))"
+ result = f'({col}.fill_null("").startswith({_as_literal(value)}))'
elif operator == "ends_with":
- result = f"({col}.endswith({_as_literal(value)}))"
+ result = f'({col}.fill_null("").endswith({_as_literal(value)}))'
elif operator == "in" or operator == "not_in":
expr = f"({col}.isin({_list_of_strings(value)}))"
result = expr if operator == "in" else f"~{expr}"
@@ -586,6 +592,8 @@ def python_print_ibis(
if where.negate:
result = f"~({result})"
+ if operator in {"contains", "regex", "starts_with", "ends_with"}:
+ result = f"({result} & {col}.notnull())"
return result
diff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
index d5207939f5..fd6997d20d 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import ast
import datetime
import string
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Literal, cast
import narwhals.stable.v2 as nw
import pytest
@@ -31,6 +31,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
FilterGroup,
FilterRowsTransform,
GroupByTransform,
+ Operator,
PivotTransform,
RenameColumnTransform,
SampleRowsTransform,
@@ -271,6 +272,118 @@ transformations_strategy = st.builds(
)
+def _assert_string_filter_matches_runtime(
+ backend: str,
+ strings: list[str | None],
+ operator: Operator,
+ value: str,
+ negate: bool,
+ operation: Literal["keep_rows", "remove_rows"],
+) -> None:
+ data = {"strings": strings, "row_id": list(range(len(strings)))}
+ if backend.startswith("pandas"):
+ pd = pytest.importorskip("pandas")
+ df = pd.DataFrame(data).astype(
+ {"strings": "object" if backend == "pandas-object" else "string"}
+ )
+ printer = python_print_pandas
+ namespace = {"pd": pd, "df": df}
+ elif backend == "polars":
+ pl = pytest.importorskip("polars")
+ df = pl.DataFrame(data, schema_overrides={"strings": pl.String})
+ printer = python_print_polars
+ namespace = {"pl": pl, "df": df}
+ else:
+ ibis = pytest.importorskip("ibis")
+ pytest.importorskip("duckdb")
+ df = ibis.memtable(
+ data, schema={"strings": "string", "row_id": "int64"}
+ )
+ printer = python_print_ibis
+ namespace = {"ibis": ibis, "df": df}
+
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation=operation,
+ where=FilterGroup(
+ operator="and",
+ children=[
+ FilterCondition(
+ column_id="strings",
+ operator=operator,
+ value=value,
+ negate=negate,
+ )
+ ],
+ ),
+ )
+ expected = _apply_transforms(
+ nw.from_native(df).lazy(),
+ NarwhalsTransformHandler(),
+ Transformations([transform]),
+ ).collect()
+ code = python_print_transforms("df", list(data), [transform], printer)
+ exec(code, namespace)
+ actual = nw.from_native(namespace["df_next"]).lazy().collect()
+ assert actual.to_dict(as_series=False) == expected.to_dict(as_series=False)
+
+
+ "backend", ["pandas-object", "pandas-string", "polars", "ibis"]
+)
+ "operator", ["contains", "regex", "starts_with", "ends_with"]
+)
[email protected]("value", ["", "a", "a.c"])
[email protected]("negate", [False, True])
[email protected]("operation", ["keep_rows", "remove_rows"])
+def test_print_code_string_filters_with_nulls(
+ backend: str,
+ operator: Operator,
+ value: str,
+ negate: bool,
+ operation: Literal["keep_rows", "remove_rows"],
+) -> None:
+ _assert_string_filter_matches_runtime(
+ backend,
+ [None, "", "abc", "a.c", "cba", "zzz"],
+ operator,
+ value,
+ negate,
+ operation,
+ )
+
+
+ "backend", ["pandas-object", "pandas-string", "polars", "ibis"]
+)
+@given(
+ strings=st.lists(
+ st.one_of(st.none(), st.text(alphabet="abc .", max_size=10)),
+ min_size=1,
+ max_size=10,
+ ),
+ operator=st.sampled_from(
+ ["contains", "regex", "starts_with", "ends_with"]
+ ),
+ value=st.text(alphabet="abc.", max_size=3),
+ negate=st.booleans(),
+ operation=st.sampled_from(["keep_rows", "remove_rows"]),
+)
+@settings(deadline=None)
+def test_print_code_string_filters_with_generated_nulls(
+ backend: str,
+ strings: list[str | None],
+ operator: Operator,
+ value: str,
+ negate: bool,
+ operation: Literal["keep_rows", "remove_rows"],
+) -> None:
+ _assert_string_filter_matches_runtime(
+ backend, strings, operator, value, negate, operation
+ )
+
+
def _validate_code(code: str):
try:
ast.parse(code)