A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
handle nans and infs in table filter (#7402)
Shahmir Varqha3214447adc2c
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex 0a25dbf4bf..9e5e0c71b6 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -30,6 +30,11 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (TransformHandler,UniqueTransform,)+from marimo._plugins.ui._impl.tables.narwhals_table import (+ NAN_VALUE,+ NEGATIVE_INF,+ POSITIVE_INF,+)from marimo._utils.assert_never import assert_neverfrom marimo._utils.narwhals_utils import collect_and_preserve_type@@ -122,32 +127,52 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):filter_expr: nw.Expr | None = Nonedef convert_value(v: Any, converter: Callable[[str], Any]) -> Any:- # Convert a value whether it's a list or single value+ """+ Convert a value whether it's a list or single value.+ Ignore None as they usually raise errors when converted+ """if isinstance(v, (tuple, list)):- return [converter(str(item)) for item in v]+ return [+ converter(str(item)) if item is not None else None+ for item in v+ ]+ if v is None:+ return Nonereturn converter(str(v))for condition in transform.where:# Don't convert to string if already a string or int# Narwhals col() can handle both strings and integerscolumn = col(condition.column_id)+ column_name = str(condition.column_id)value = condition.value- # For polars, we need to convert the values based on dtypenative_df = df.to_native()- if _is_polars_dataframe_or_lazyframe(native_df):- import polars as pl+ dtype = df.collect_schema().get(column_name)- dtype = native_df.collect_schema()[str(condition.column_id)]- if dtype == pl.Datetime:+ # For polars, we need to convert the values based on dtype+ if _is_polars_dataframe_or_lazyframe(native_df):+ if dtype == nw.Datetime:value = convert_value(value, datetime.datetime.fromisoformat)- elif dtype == pl.Date:+ elif dtype == nw.Date:value = convert_value(value, datetime.date.fromisoformat)- elif dtype == pl.Time:+ elif dtype == nw.Time:value = convert_value(value, datetime.time.fromisoformat)+ # If the value includes NaNs or infs, we convert to floats so the filters apply correctly+ if (+ isinstance(value, tuple)+ and any(+ token in value+ for token in [NAN_VALUE, POSITIVE_INF, NEGATIVE_INF]+ )+ and dtype is not None+ and dtype.is_float() # Note: this doesn't cover Object types for pandas+ ):+ value = convert_value(value, float)+# Build the expression based on the operatorcondition_expr: nw.Exprif condition.operator == "==":diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.pyindex 4d67d219a5..ba3d79ae50 100644--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport datetimeimport functoolsimport io+import mathfrom functools import cached_propertyfrom typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast@@ -45,6 +46,12 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()UNSTABLE_API_WARNING = "`Series.hist` is being called from the stable API although considered an unstable feature."+# Standardize this across libraries+# It should match the table value as closely as possible+NAN_VALUE = "NaN"+POSITIVE_INF = str(float("inf"))+NEGATIVE_INF = str(float("-inf"))+class NarwhalsTableManager(TableManager[@@ -218,10 +225,22 @@ class NarwhalsTableManager(]result = _calculate_top_k_rows(frame)- return [- (unwrap_py_scalar(row[0]), int(unwrap_py_scalar(row[1])))- for row in result.rows()- ]+ value_counts: list[tuple[Any, int]] = []++ # NaNs and Infs serialize to null, which isn't distingushable from normal nulls+ # so instead we set to string values+ for row in result.rows():+ value = unwrap_py_scalar(row[0])+ count = int(unwrap_py_scalar(row[1]))+ if isinstance(value, float) and math.isnan(value):+ value = NAN_VALUE+ elif isinstance(value, float) and math.isinf(value) and value > 0:+ value = POSITIVE_INF+ elif isinstance(value, float) and math.isinf(value) and value < 0:+ value = NEGATIVE_INF+ value_counts.append((value, count))++ return value_counts@staticmethoddef is_type(value: Any) -> bool:diff --git a/marimo/_smoke_tests/tables/nans.py b/marimo/_smoke_tests/tables/nans.pyindex 5db671834d..2ab2c21253 100644--- a/marimo/_smoke_tests/tables/nans.py+++ b/marimo/_smoke_tests/tables/nans.py@@ -1,6 +1,6 @@import marimo-__generated_with = "0.17.6"+__generated_with = "0.18.2"app = marimo.App(width="medium")@@ -31,10 +31,18 @@ def _(np, pd, pl):@app.celldef _(mo, np, pd):- pandas_df = pd.DataFrame(+ pandas_df_obj = pd.DataFrame({"nans": [1, None, pd.NaT, np.nan, pd.NA, np.inf, -np.inf]})- mo.vstack([mo.plain(pandas_df), pandas_df])+ mo.vstack([mo.plain(pandas_df_obj), pandas_df_obj])+ return+++def _(pd):+ # Filtering NaNs work for float types+ pandas_nan_float = pd.DataFrame({"nans": [1, 2, 3, 4, float("nan")]})+ pandas_nan_floatreturndiff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.pyindex 6a57950259..ee07711101 100644--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py@@ -33,6 +33,11 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (TransformType,UniqueTransform,)+from marimo._plugins.ui._impl.tables.narwhals_table import (+ NAN_VALUE,+ NEGATIVE_INF,+ POSITIVE_INF,+)from marimo._utils.narwhals_utils import is_narwhals_lazyframe, make_lazyfrom tests._data.mocks import create_dataframes@@ -76,6 +81,39 @@ def assert_frame_equal(a: DataFrameType, b: DataFrameType) -> None:assert nw_a.to_dict(as_series=False) == nw_b.to_dict(as_series=False)+def assert_frame_equal_with_nans(a: DataFrameType, b: DataFrameType) -> None:+ """+ Assert two dataframes are equal, treating NaNs in the same locations as equal.+ """+ import math++ nw_a = collect_df(a)+ nw_b = collect_df(b)++ dict_a = nw_a.to_dict(as_series=False)+ dict_b = nw_b.to_dict(as_series=False)++ assert dict_a.keys() == dict_b.keys(), "DataFrame columns do not match."++ for col in dict_a:+ values_a = dict_a[col]+ values_b = dict_b[col]+ assert len(values_a) == len(values_b), (+ f"Length mismatch in column {col}"+ )+ for idx, (val_a, val_b) in enumerate(zip(values_a, values_b)):+ both_nan = (+ isinstance(val_a, float)+ and isinstance(val_b, float)+ and math.isnan(val_a)+ and math.isnan(val_b)+ )+ if not (val_a == val_b or both_nan):+ raise AssertionError(+ f"DataFrame values differ at column '{col}', row {idx}: {val_a} != {val_b}"+ )++def assert_frame_not_equal(df1: DataFrameType, df2: DataFrameType) -> None:with pytest.raises(AssertionError):assert_frame_equal(df1, df2)@@ -1164,3 +1202,244 @@ class TestTransformHandler:)result = apply(df, in_transform)assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"nulls": [1, 2, 3, None]}, include=["pandas"]+ ),+ create_test_dataframes(+ {"nulls": [float("nan")]}, include=["pandas"]+ ),+ )+ ),+ )+ def test_filter_rows_nulls_pandas(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NAN_VALUE],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)++ @staticmethod+ @pytest.mark.xfail(+ reason="NaN filtering for object dtypes in pandas aren't implemented"+ )+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"nulls": [1, 2, 3, None, "hello"]}, include=["pandas"]+ ),+ create_test_dataframes(+ {"nulls": [float("nan")]}, include=["pandas"]+ ),+ )+ ),+ )+ def test_filter_rows_null_pandas_object(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NAN_VALUE],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"nulls": [1, 2, 3, float("nan")]},+ exclude=["pandas", "ibis"], # Ibis serializes nans to None+ strict=False,+ ),+ create_test_dataframes(+ {"nulls": [float("nan")]}, exclude=["pandas", "ibis"]+ ),+ )+ ),+ )+ def test_filter_rows_nulls_others(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NAN_VALUE],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"nulls": [1, 2, 3, float("nan"), float("inf")]},+ strict=False,+ ),+ create_test_dataframes({"nulls": [float("inf")]}),+ )+ ),+ )+ def test_filter_rows_infs(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[POSITIVE_INF],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"nulls": [1, float("nan"), float("inf"), float("-inf")]},+ strict=False,+ ),+ create_test_dataframes({"nulls": [float("-inf")]}),+ )+ ),+ )+ def test_filter_rows_neg_infs(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NEGATIVE_INF],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {+ "nulls": [+ 1,+ float("nan"),+ float("inf"),+ float("-inf"),+ None,+ ]+ },+ include=["pandas"],+ ),+ create_test_dataframes(+ {"nulls": [float("nan"), float("inf"), None]},+ include=["pandas"],+ ),+ )+ ),+ )+ def test_filter_rows_infs_and_nulls_pandas(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NAN_VALUE, POSITIVE_INF, None],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {+ "nulls": [+ 1,+ float("nan"),+ float("inf"),+ float("-inf"),+ None,+ ]+ },+ exclude=["pandas", "ibis"], # Ibis serializes nans to None+ strict=False,+ ),+ create_test_dataframes(+ {"nulls": [float("nan"), float("inf"), None]},+ exclude=["pandas", "ibis"],+ ),+ )+ ),+ )+ def test_filter_rows_infs_and_nulls_others(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="nulls",+ operator="in",+ value=[NAN_VALUE, POSITIVE_INF, None],+ )+ ],+ )+ result = apply(df, in_transform)+ assert_frame_equal_with_nans(result, expected)diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.pyindex fd69e59855..48972b6497 100644--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py@@ -17,6 +17,9 @@ from marimo._plugins.ui._impl.input import buttonfrom marimo._plugins.ui._impl.table import SortArgsfrom marimo._plugins.ui._impl.tables.format import FormatMappingfrom marimo._plugins.ui._impl.tables.narwhals_table import (+ NAN_VALUE,+ NEGATIVE_INF,+ POSITIVE_INF,NarwhalsTableManager,)from marimo._plugins.ui._impl.tables.table_manager import (@@ -1416,12 +1419,15 @@ def test_calculate_top_k_rows(df: Any) -> None:manager = NarwhalsTableManager.from_dataframe(df)result = manager.calculate_top_k_rows("A", 10)normalized_result = _normalize_result(result)- assert normalized_result == [(3, 3), (None, 2), (1, 1), (2, 1)]++ # Pandas considers None as nan sometimes+ none_value = NAN_VALUE if nw.dependencies.is_pandas_dataframe(df) else None+ assert normalized_result == [(3, 3), (none_value, 2), (1, 1), (2, 1)]# Test with limitresult = manager.calculate_top_k_rows("A", 2)normalized_result = _normalize_result(result)- assert normalized_result == [(3, 3), (None, 2)]+ assert normalized_result == [(3, 3), (none_value, 2)]@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")@@ -1451,12 +1457,85 @@ def test_calculate_top_k_rows_dicts(df: Any) -> None:assert result == [({"a": 1, "b": 2}, 2), ({"a": 3, "b": 4}, 1)][email protected](not HAS_DEPS, reason="optional dependencies not installed")+ "df",+ create_dataframes(+ {"A": [float("nan"), float("nan"), float("nan"), 1.0, 2.0, 2.0]},+ include=SUPPORTED_LIBS,+ ),+)+def test_calculate_top_k_rows_with_nan(df: Any) -> None:+ """Test that NaN values are converted to NAN_VALUE string in calculate_top_k_rows."""++ manager = NarwhalsTableManager.from_dataframe(df)+ result = manager.calculate_top_k_rows("A", 10)++ # Ibis serializes nans as None+ none_value = None if nw.dependencies.is_ibis_table(df) else NAN_VALUE+ # NaN values should be converted to NAN_VALUE string+ assert result == [(none_value, 3), (2.0, 2), (1.0, 1)]++[email protected](not HAS_DEPS, reason="optional dependencies not installed")+ "df",+ create_dataframes(+ {+ "A": [+ float("nan"),+ float("nan"),+ float("inf"),+ float("inf"),+ float("-inf"),+ 1.0,+ 2.0,+ ]+ },+ include=SUPPORTED_LIBS,+ ),+)+def test_calculate_top_k_rows_with_all_special_floats(df: Any) -> None:+ """Test that NaN, positive infinity, and negative infinity are all handled correctly."""+ manager = NarwhalsTableManager.from_dataframe(df)+ result = manager.calculate_top_k_rows("A", 10)++ # Ibis serializes nans as None+ none_value = None if nw.dependencies.is_ibis_table(df) else NAN_VALUE++ pandas_pyarrow_ibis = (+ nw.dependencies.is_pandas_dataframe(df)+ or nw.dependencies.is_pyarrow_table(df)+ or nw.dependencies.is_ibis_table(df)+ )++ # Different libraries order NaNs and Infs differently+ if pandas_pyarrow_ibis:+ assert result == [+ (none_value, 2),+ (POSITIVE_INF, 2),+ (NEGATIVE_INF, 1),+ (1.0, 1),+ (2.0, 1),+ ]+ else:+ assert result == [+ (POSITIVE_INF, 2),+ (NAN_VALUE, 2),+ (NEGATIVE_INF, 1),+ (1.0, 1),+ (2.0, 1),+ ]++def _normalize_result(result: list[tuple[Any, int]]) -> list[tuple[Any, int]]:"""Normalize None and NaN values for comparison."""- return [- (None if val is None or isnan(val) else val, count)- for val, count in result- ]+ out: list[tuple[Any, int]] = []+ for val, count in result:+ if isinstance(val, (float, int)) and isnan(val):+ val = NAN_VALUE+ out.append((val, count))+ return out@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")