Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

handle nans and infs in table filter (#7402)

Shahmir Varqha3214447adc2c
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index 0a25dbf4bf..9e5e0c71b6 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -30,6 +30,11 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
TransformHandler,
UniqueTransform,
)
+from marimo._plugins.ui._impl.tables.narwhals_table import (
+ NAN_VALUE,
+ NEGATIVE_INF,
+ POSITIVE_INF,
+)
from marimo._utils.assert_never import assert_never
from marimo._utils.narwhals_utils import collect_and_preserve_type
@@ -122,32 +127,52 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
filter_expr: nw.Expr | None = None
def convert_value(v: Any, converter: Callable[[str], Any]) -> Any:
- # Convert a value whether it's a list or single value
+ """
+ Convert a value whether it's a list or single value.
+ Ignore None as they usually raise errors when converted
+ """
if isinstance(v, (tuple, list)):
- return [converter(str(item)) for item in v]
+ return [
+ converter(str(item)) if item is not None else None
+ for item in v
+ ]
+ if v is None:
+ return None
return converter(str(v))
for condition in transform.where:
# Don't convert to string if already a string or int
# Narwhals col() can handle both strings and integers
column = col(condition.column_id)
+ column_name = str(condition.column_id)
value = condition.value
- # For polars, we need to convert the values based on dtype
native_df = df.to_native()
- if _is_polars_dataframe_or_lazyframe(native_df):
- import polars as pl
+ dtype = df.collect_schema().get(column_name)
- dtype = native_df.collect_schema()[str(condition.column_id)]
- if dtype == pl.Datetime:
+ # For polars, we need to convert the values based on dtype
+ if _is_polars_dataframe_or_lazyframe(native_df):
+ if dtype == nw.Datetime:
value = convert_value(
value, datetime.datetime.fromisoformat
)
- elif dtype == pl.Date:
+ elif dtype == nw.Date:
value = convert_value(value, datetime.date.fromisoformat)
- elif dtype == pl.Time:
+ elif dtype == nw.Time:
value = convert_value(value, datetime.time.fromisoformat)
+ # If the value includes NaNs or infs, we convert to floats so the filters apply correctly
+ if (
+ isinstance(value, tuple)
+ and any(
+ token in value
+ for token in [NAN_VALUE, POSITIVE_INF, NEGATIVE_INF]
+ )
+ and dtype is not None
+ and dtype.is_float() # Note: this doesn't cover Object types for pandas
+ ):
+ value = convert_value(value, float)
+
# Build the expression based on the operator
condition_expr: nw.Expr
if condition.operator == "==":
diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
index 4d67d219a5..ba3d79ae50 100644
--- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py
+++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import datetime
import functools
import io
+import math
from functools import cached_property
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
@@ -45,6 +46,12 @@ if TYPE_CHECKING:
LOGGER = _loggers.marimo_logger()
UNSTABLE_API_WARNING = "`Series.hist` is being called from the stable API although considered an unstable feature."
+# Standardize this across libraries
+# It should match the table value as closely as possible
+NAN_VALUE = "NaN"
+POSITIVE_INF = str(float("inf"))
+NEGATIVE_INF = str(float("-inf"))
+
class NarwhalsTableManager(
TableManager[
@@ -218,10 +225,22 @@ class NarwhalsTableManager(
]
result = _calculate_top_k_rows(frame)
- return [
- (unwrap_py_scalar(row[0]), int(unwrap_py_scalar(row[1])))
- for row in result.rows()
- ]
+ value_counts: list[tuple[Any, int]] = []
+
+ # NaNs and Infs serialize to null, which isn't distingushable from normal nulls
+ # so instead we set to string values
+ for row in result.rows():
+ value = unwrap_py_scalar(row[0])
+ count = int(unwrap_py_scalar(row[1]))
+ if isinstance(value, float) and math.isnan(value):
+ value = NAN_VALUE
+ elif isinstance(value, float) and math.isinf(value) and value > 0:
+ value = POSITIVE_INF
+ elif isinstance(value, float) and math.isinf(value) and value < 0:
+ value = NEGATIVE_INF
+ value_counts.append((value, count))
+
+ return value_counts
@staticmethod
def is_type(value: Any) -> bool:
diff --git a/marimo/_smoke_tests/tables/nans.py b/marimo/_smoke_tests/tables/nans.py
index 5db671834d..2ab2c21253 100644
--- a/marimo/_smoke_tests/tables/nans.py
+++ b/marimo/_smoke_tests/tables/nans.py
@@ -1,6 +1,6 @@
import marimo
-__generated_with = "0.17.6"
+__generated_with = "0.18.2"
app = marimo.App(width="medium")
@@ -31,10 +31,18 @@ def _(np, pd, pl):
@app.cell
def _(mo, np, pd):
- pandas_df = pd.DataFrame(
+ pandas_df_obj = pd.DataFrame(
{"nans": [1, None, pd.NaT, np.nan, pd.NA, np.inf, -np.inf]}
)
- mo.vstack([mo.plain(pandas_df), pandas_df])
+ mo.vstack([mo.plain(pandas_df_obj), pandas_df_obj])
+ return
+
+
+def _(pd):
+ # Filtering NaNs work for float types
+ pandas_nan_float = pd.DataFrame({"nans": [1, 2, 3, 4, float("nan")]})
+ pandas_nan_float
return
diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
index 6a57950259..ee07711101 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
@@ -33,6 +33,11 @@ from marimo._plugins.ui._impl.dataframes.transforms.types import (
TransformType,
UniqueTransform,
)
+from marimo._plugins.ui._impl.tables.narwhals_table import (
+ NAN_VALUE,
+ NEGATIVE_INF,
+ POSITIVE_INF,
+)
from marimo._utils.narwhals_utils import is_narwhals_lazyframe, make_lazy
from tests._data.mocks import create_dataframes
@@ -76,6 +81,39 @@ def assert_frame_equal(a: DataFrameType, b: DataFrameType) -> None:
assert nw_a.to_dict(as_series=False) == nw_b.to_dict(as_series=False)
+def assert_frame_equal_with_nans(a: DataFrameType, b: DataFrameType) -> None:
+ """
+ Assert two dataframes are equal, treating NaNs in the same locations as equal.
+ """
+ import math
+
+ nw_a = collect_df(a)
+ nw_b = collect_df(b)
+
+ dict_a = nw_a.to_dict(as_series=False)
+ dict_b = nw_b.to_dict(as_series=False)
+
+ assert dict_a.keys() == dict_b.keys(), "DataFrame columns do not match."
+
+ for col in dict_a:
+ values_a = dict_a[col]
+ values_b = dict_b[col]
+ assert len(values_a) == len(values_b), (
+ f"Length mismatch in column {col}"
+ )
+ for idx, (val_a, val_b) in enumerate(zip(values_a, values_b)):
+ both_nan = (
+ isinstance(val_a, float)
+ and isinstance(val_b, float)
+ and math.isnan(val_a)
+ and math.isnan(val_b)
+ )
+ if not (val_a == val_b or both_nan):
+ raise AssertionError(
+ f"DataFrame values differ at column '{col}', row {idx}: {val_a} != {val_b}"
+ )
+
+
def assert_frame_not_equal(df1: DataFrameType, df2: DataFrameType) -> None:
with pytest.raises(AssertionError):
assert_frame_equal(df1, df2)
@@ -1164,3 +1202,244 @@ class TestTransformHandler:
)
result = apply(df, in_transform)
assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"nulls": [1, 2, 3, None]}, include=["pandas"]
+ ),
+ create_test_dataframes(
+ {"nulls": [float("nan")]}, include=["pandas"]
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_nulls_pandas(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NAN_VALUE],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
+
+ @staticmethod
+ @pytest.mark.xfail(
+ reason="NaN filtering for object dtypes in pandas aren't implemented"
+ )
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"nulls": [1, 2, 3, None, "hello"]}, include=["pandas"]
+ ),
+ create_test_dataframes(
+ {"nulls": [float("nan")]}, include=["pandas"]
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_null_pandas_object(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NAN_VALUE],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"nulls": [1, 2, 3, float("nan")]},
+ exclude=["pandas", "ibis"], # Ibis serializes nans to None
+ strict=False,
+ ),
+ create_test_dataframes(
+ {"nulls": [float("nan")]}, exclude=["pandas", "ibis"]
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_nulls_others(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NAN_VALUE],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"nulls": [1, 2, 3, float("nan"), float("inf")]},
+ strict=False,
+ ),
+ create_test_dataframes({"nulls": [float("inf")]}),
+ )
+ ),
+ )
+ def test_filter_rows_infs(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[POSITIVE_INF],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"nulls": [1, float("nan"), float("inf"), float("-inf")]},
+ strict=False,
+ ),
+ create_test_dataframes({"nulls": [float("-inf")]}),
+ )
+ ),
+ )
+ def test_filter_rows_neg_infs(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NEGATIVE_INF],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {
+ "nulls": [
+ 1,
+ float("nan"),
+ float("inf"),
+ float("-inf"),
+ None,
+ ]
+ },
+ include=["pandas"],
+ ),
+ create_test_dataframes(
+ {"nulls": [float("nan"), float("inf"), None]},
+ include=["pandas"],
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_infs_and_nulls_pandas(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NAN_VALUE, POSITIVE_INF, None],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {
+ "nulls": [
+ 1,
+ float("nan"),
+ float("inf"),
+ float("-inf"),
+ None,
+ ]
+ },
+ exclude=["pandas", "ibis"], # Ibis serializes nans to None
+ strict=False,
+ ),
+ create_test_dataframes(
+ {"nulls": [float("nan"), float("inf"), None]},
+ exclude=["pandas", "ibis"],
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_infs_and_nulls_others(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="nulls",
+ operator="in",
+ value=[NAN_VALUE, POSITIVE_INF, None],
+ )
+ ],
+ )
+ result = apply(df, in_transform)
+ assert_frame_equal_with_nans(result, expected)
diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals.py b/tests/_plugins/ui/_impl/tables/test_narwhals.py
index fd69e59855..48972b6497 100644
--- a/tests/_plugins/ui/_impl/tables/test_narwhals.py
+++ b/tests/_plugins/ui/_impl/tables/test_narwhals.py
@@ -17,6 +17,9 @@ from marimo._plugins.ui._impl.input import button
from marimo._plugins.ui._impl.table import SortArgs
from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.narwhals_table import (
+ NAN_VALUE,
+ NEGATIVE_INF,
+ POSITIVE_INF,
NarwhalsTableManager,
)
from marimo._plugins.ui._impl.tables.table_manager import (
@@ -1416,12 +1419,15 @@ def test_calculate_top_k_rows(df: Any) -> None:
manager = NarwhalsTableManager.from_dataframe(df)
result = manager.calculate_top_k_rows("A", 10)
normalized_result = _normalize_result(result)
- assert normalized_result == [(3, 3), (None, 2), (1, 1), (2, 1)]
+
+ # Pandas considers None as nan sometimes
+ none_value = NAN_VALUE if nw.dependencies.is_pandas_dataframe(df) else None
+ assert normalized_result == [(3, 3), (none_value, 2), (1, 1), (2, 1)]
# Test with limit
result = manager.calculate_top_k_rows("A", 2)
normalized_result = _normalize_result(result)
- assert normalized_result == [(3, 3), (None, 2)]
+ assert normalized_result == [(3, 3), (none_value, 2)]
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
@@ -1451,12 +1457,85 @@ def test_calculate_top_k_rows_dicts(df: Any) -> None:
assert result == [({"a": 1, "b": 2}, 2), ({"a": 3, "b": 4}, 1)]
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+ "df",
+ create_dataframes(
+ {"A": [float("nan"), float("nan"), float("nan"), 1.0, 2.0, 2.0]},
+ include=SUPPORTED_LIBS,
+ ),
+)
+def test_calculate_top_k_rows_with_nan(df: Any) -> None:
+ """Test that NaN values are converted to NAN_VALUE string in calculate_top_k_rows."""
+
+ manager = NarwhalsTableManager.from_dataframe(df)
+ result = manager.calculate_top_k_rows("A", 10)
+
+ # Ibis serializes nans as None
+ none_value = None if nw.dependencies.is_ibis_table(df) else NAN_VALUE
+ # NaN values should be converted to NAN_VALUE string
+ assert result == [(none_value, 3), (2.0, 2), (1.0, 1)]
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+ "df",
+ create_dataframes(
+ {
+ "A": [
+ float("nan"),
+ float("nan"),
+ float("inf"),
+ float("inf"),
+ float("-inf"),
+ 1.0,
+ 2.0,
+ ]
+ },
+ include=SUPPORTED_LIBS,
+ ),
+)
+def test_calculate_top_k_rows_with_all_special_floats(df: Any) -> None:
+ """Test that NaN, positive infinity, and negative infinity are all handled correctly."""
+ manager = NarwhalsTableManager.from_dataframe(df)
+ result = manager.calculate_top_k_rows("A", 10)
+
+ # Ibis serializes nans as None
+ none_value = None if nw.dependencies.is_ibis_table(df) else NAN_VALUE
+
+ pandas_pyarrow_ibis = (
+ nw.dependencies.is_pandas_dataframe(df)
+ or nw.dependencies.is_pyarrow_table(df)
+ or nw.dependencies.is_ibis_table(df)
+ )
+
+ # Different libraries order NaNs and Infs differently
+ if pandas_pyarrow_ibis:
+ assert result == [
+ (none_value, 2),
+ (POSITIVE_INF, 2),
+ (NEGATIVE_INF, 1),
+ (1.0, 1),
+ (2.0, 1),
+ ]
+ else:
+ assert result == [
+ (POSITIVE_INF, 2),
+ (NAN_VALUE, 2),
+ (NEGATIVE_INF, 1),
+ (1.0, 1),
+ (2.0, 1),
+ ]
+
+
def _normalize_result(result: list[tuple[Any, int]]) -> list[tuple[Any, int]]:
"""Normalize None and NaN values for comparison."""
- return [
- (None if val is None or isnan(val) else val, count)
- for val, count in result
- ]
+ out: list[tuple[Any, int]] = []
+ for val, count in result:
+ if isinstance(val, (float, int)) and isnan(val):
+ val = NAN_VALUE
+ out.append((val, count))
+ return out
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")