A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: preserve null handling in generated filters (#11036)
Kiran Gadhave719f967b9dff
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.pyindex d452437e36..ced183dc58 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/print_code.py@@ -49,13 +49,13 @@ def python_print_pandas(elif operator in [">", ">=", "<", "<="]:result = f"{col} {operator} {_as_literal(value)}"elif operator == "contains":- result = f"{col}.str.contains({_as_literal(value)}, regex=False)"+ result = f'{col}.fillna("").str.contains({_as_literal(value)}, regex=False)'elif operator == "regex":- result = f"{col}.str.contains({_as_literal(value)}, regex=True)"+ result = f'{col}.fillna("").str.contains({_as_literal(value)}, regex=True)'elif operator == "starts_with":- result = f"{col}.str.startswith({_as_literal(value)})"+ result = f'{col}.fillna("").str.startswith({_as_literal(value)})'elif operator == "ends_with":- result = f"{col}.str.endswith({_as_literal(value)})"+ result = f'{col}.fillna("").str.endswith({_as_literal(value)})'elif operator == "in" or operator == "not_in":expr = f"{col}.isin({_list_of_strings(value)})"result = expr if operator == "in" else f"~{expr}"@@ -81,6 +81,8 @@ def python_print_pandas(if where.negate:result = f"~({result})"+ if operator in {"contains", "regex", "starts_with", "ends_with"}:+ result = f"({result} & {col}.notna())"return result@@ -317,13 +319,15 @@ def python_print_polars(elif operator in [">", ">=", "<", "<="]:result = f"{col} {operator} {_as_literal(value)}"elif operator == "contains":- result = f"{col}.str.contains({_as_literal(value)}, literal=True)"+ result = f'{col}.fill_null("").str.contains({_as_literal(value)}, literal=True)'elif operator == "regex":- result = f"{col}.str.contains({_as_literal(value)}, literal=False)"+ result = f'{col}.fill_null("").str.contains({_as_literal(value)}, literal=False)'elif operator == "starts_with":- result = f"{col}.str.starts_with({_as_literal(value)})"+ result = (+ f'{col}.fill_null("").str.starts_with({_as_literal(value)})'+ )elif operator == "ends_with":- result = f"{col}.str.ends_with({_as_literal(value)})"+ result = f'{col}.fill_null("").str.ends_with({_as_literal(value)})'elif operator == "in" or operator == "not_in":expr = f"{col}.is_in({_list_of_strings(value)})"result = expr if operator == "in" else f"~{expr}"@@ -349,6 +353,8 @@ def python_print_polars(if where.negate:result = f"~({result})"+ if operator in {"contains", "regex", "starts_with", "ends_with"}:+ result = f"({result} & {col}.is_not_null())"return result@@ -554,13 +560,13 @@ def python_print_ibis(elif operator in [">", ">=", "<", "<="]:result = f"({col} {operator} {_as_literal(value)})"elif operator == "contains":- result = f"({col}.contains({_as_literal(value)}))"+ result = f'({col}.fill_null("").contains({_as_literal(value)}))'elif operator == "regex":- result = f"({col}.re_search({_as_literal(value)}))"+ result = f'({col}.fill_null("").re_search({_as_literal(value)}))'elif operator == "starts_with":- result = f"({col}.startswith({_as_literal(value)}))"+ result = f'({col}.fill_null("").startswith({_as_literal(value)}))'elif operator == "ends_with":- result = f"({col}.endswith({_as_literal(value)}))"+ result = f'({col}.fill_null("").endswith({_as_literal(value)}))'elif operator == "in" or operator == "not_in":expr = f"({col}.isin({_list_of_strings(value)}))"result = expr if operator == "in" else f"~{expr}"@@ -586,6 +592,8 @@ def python_print_ibis(if where.negate:result = f"~({result})"+ if operator in {"contains", "regex", "starts_with", "ends_with"}:+ result = f"({result} & {col}.notnull())"return resultdiff --git a/tests/_plugins/ui/_impl/dataframes/test_print_code.py b/tests/_plugins/ui/_impl/dataframes/test_print_code.pyindex d5207939f5..fd6997d20d 100644--- a/tests/_plugins/ui/_impl/dataframes/test_print_code.py+++ b/tests/_plugins/ui/_impl/dataframes/test_print_code.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport astimport datetimeimport string-from typing import TYPE_CHECKING, cast+from typing import TYPE_CHECKING, Literal, castimport narwhals.stable.v2 as nwimport pytest@@ -31,6 +31,7 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (FilterGroup,FilterRowsTransform,GroupByTransform,+ Operator,PivotTransform,RenameColumnTransform,SampleRowsTransform,@@ -271,6 +272,118 @@ transformations_strategy = st.builds()+def _assert_string_filter_matches_runtime(+ backend: str,+ strings: list[str | None],+ operator: Operator,+ value: str,+ negate: bool,+ operation: Literal["keep_rows", "remove_rows"],+) -> None:+ data = {"strings": strings, "row_id": list(range(len(strings)))}+ if backend.startswith("pandas"):+ pd = pytest.importorskip("pandas")+ df = pd.DataFrame(data).astype(+ {"strings": "object" if backend == "pandas-object" else "string"}+ )+ printer = python_print_pandas+ namespace = {"pd": pd, "df": df}+ elif backend == "polars":+ pl = pytest.importorskip("polars")+ df = pl.DataFrame(data, schema_overrides={"strings": pl.String})+ printer = python_print_polars+ namespace = {"pl": pl, "df": df}+ else:+ ibis = pytest.importorskip("ibis")+ pytest.importorskip("duckdb")+ df = ibis.memtable(+ data, schema={"strings": "string", "row_id": "int64"}+ )+ printer = python_print_ibis+ namespace = {"ibis": ibis, "df": df}++ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation=operation,+ where=FilterGroup(+ operator="and",+ children=[+ FilterCondition(+ column_id="strings",+ operator=operator,+ value=value,+ negate=negate,+ )+ ],+ ),+ )+ expected = _apply_transforms(+ nw.from_native(df).lazy(),+ NarwhalsTransformHandler(),+ Transformations([transform]),+ ).collect()+ code = python_print_transforms("df", list(data), [transform], printer)+ exec(code, namespace)+ actual = nw.from_native(namespace["df_next"]).lazy().collect()+ assert actual.to_dict(as_series=False) == expected.to_dict(as_series=False)+++ "backend", ["pandas-object", "pandas-string", "polars", "ibis"]+)+ "operator", ["contains", "regex", "starts_with", "ends_with"]+)[email protected]("value", ["", "a", "a.c"])[email protected]("negate", [False, True])[email protected]("operation", ["keep_rows", "remove_rows"])+def test_print_code_string_filters_with_nulls(+ backend: str,+ operator: Operator,+ value: str,+ negate: bool,+ operation: Literal["keep_rows", "remove_rows"],+) -> None:+ _assert_string_filter_matches_runtime(+ backend,+ [None, "", "abc", "a.c", "cba", "zzz"],+ operator,+ value,+ negate,+ operation,+ )+++ "backend", ["pandas-object", "pandas-string", "polars", "ibis"]+)+@given(+ strings=st.lists(+ st.one_of(st.none(), st.text(alphabet="abc .", max_size=10)),+ min_size=1,+ max_size=10,+ ),+ operator=st.sampled_from(+ ["contains", "regex", "starts_with", "ends_with"]+ ),+ value=st.text(alphabet="abc.", max_size=3),+ negate=st.booleans(),+ operation=st.sampled_from(["keep_rows", "remove_rows"]),+)+@settings(deadline=None)+def test_print_code_string_filters_with_generated_nulls(+ backend: str,+ strings: list[str | None],+ operator: Operator,+ value: str,+ negate: bool,+ operation: Literal["keep_rows", "remove_rows"],+) -> None:+ _assert_string_filter_matches_runtime(+ backend, strings, operator, value, negate, operation+ )++def _validate_code(code: str):try:ast.parse(code)