A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Convert decimals to float64 for filtering in tables (#8706)
Shahmir Varqha8eec31fe64ca
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.pyindex 5d9f63275a..5b53b12d26 100644--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py@@ -2,6 +2,7 @@from __future__ import annotationsimport datetime+import decimalfrom functools import reducefrom typing import TYPE_CHECKING, Any, Callable@@ -158,11 +159,12 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):elif dtype == nw.Time:value = convert_value(value, datetime.time.fromisoformat)elif dtype == nw.Object:- # Object dtype may contain date/datetime values- # (e.g., pandas with Python date objects)+ # Object dtype may contain date/datetime/Decimal values+ # (e.g., pandas stores these as Python objects)try:sample = (- df.select(col(column_name).drop_nulls())+ df.select(column_name)+ .filter(~col(column_name).is_null()).head(1).collect().get_column(column_name)@@ -181,6 +183,10 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):value = convert_value(value, datetime.time.fromisoformat)+ elif isinstance(sample[0], decimal.Decimal):+ # Cast to Float64 so Decimal values can be compared, minor precision loss+ value = convert_value(value, float)+ column = column.cast(nw.Float64)except Exception:pass@@ -195,6 +201,10 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):and dtype.is_float() # Note: this doesn't cover Object types for pandas):value = convert_value(value, float)+ elif dtype is not None and dtype.is_decimal():+ # Cast to Float64 so Decimal values can be compared, minor precision loss+ value = convert_value(value, float)+ column = column.cast(nw.Float64)# Build the expression based on the operatorcondition_expr: nw.Exprdiff --git a/marimo/_smoke_tests/tables/decimals.py b/marimo/_smoke_tests/tables/decimals.pyindex c54776a996..66c3221ec3 100644--- a/marimo/_smoke_tests/tables/decimals.py+++ b/marimo/_smoke_tests/tables/decimals.py@@ -1,36 +1,62 @@import marimo-__generated_with = "0.18.2"+__generated_with = "0.20.4"app = marimo.App(width="medium")@app.celldef _():import marimo as mo- return+ from decimal import Decimal++ return Decimal, mo@app.cell-def _():+def _(Decimal, mo, pd):import polars as pl- decimal_data = pl.DataFrame(- {"decimals": pl.Series(range(201), dtype=pl.Decimal(scale=2))}+ _data = {+ "product": ["Apple", "Banana", "Cherry", "Date"],+ "price": pl.Series(+ [+ Decimal("1.99"),+ Decimal("0.50"),+ Decimal("3.25"),+ Decimal("0.00"),+ ],+ dtype=pl.Decimal(18, 2),+ ),+ "tax": pl.Series(+ [+ Decimal("1.123456789012345678"),+ Decimal("2.987654321098765432"),+ Decimal("1.123456789012345678"),+ Decimal("-3.492354321098723632"),+ ],+ dtype=pl.Decimal(36, 18),+ ),+ "quantity": [10, 5, 8, 3],+ }++ df = pl.DataFrame(_data)+ pl_table = mo.ui.table(df, label="Polars table with decimal values")+ pd_table = mo.ui.table(+ pd.DataFrame(_data), label="Pandas table with decimal values")- decimal_data+ mo.vstack([pl_table, pd_table])return@app.cell-def _():+def _(Decimal):import pandas as pd- from decimal import Decimalpandas_decimal_data = pd.DataFrame({"decimals": [Decimal(i) / Decimal("100") for i in range(201)]})pandas_decimal_data- return+ return (pd,)if __name__ == "__main__":diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.pyindex 0a696f749c..56a5f1cc1d 100644--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py@@ -2,6 +2,7 @@from __future__ import annotationsfrom datetime import date+from decimal import Decimalfrom typing import Any, Optional, castimport narwhals.stable.v2 as nw@@ -498,6 +499,249 @@ class TestTransformHandler:result = apply(df, transform)assert_frame_equal(result, expected)+ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {"A": [Decimal("1.99"), Decimal("0.50"), Decimal("3.25")]},+ exclude=["pandas"],+ ),+ create_test_dataframes(+ {"A": [Decimal("1.99"), Decimal("3.25")]},+ exclude=["pandas"],+ ),+ )+ ),+ )+ def test_filter_rows_in_decimal(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="A",+ operator="in",+ value=["1.99", "3.25"],+ ),+ ],+ )+ result = apply(df, transform)+ assert_frame_equal(result, expected)++ @staticmethod+ @pytest.mark.parametrize(+ ("df", "expected"),+ list(+ zip(+ create_test_dataframes(+ {+ "A": [+ Decimal("0.10"),+ Decimal("0.20"),+ Decimal("0.30"),+ ]+ },+ exclude=["pandas"],+ ),+ create_test_dataframes(+ {"A": [Decimal("0.10")]},+ exclude=["pandas"],+ ),+ )+ ),+ )+ def test_filter_rows_in_decimal_not_exactly_representable_as_float(+ df: DataFrameType, expected: DataFrameType+ ) -> None:+ """0.1 is not exactly representable as a float, but filtering should still work."""+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="A",+ operator="in",+ value=["0.10"],+ ),+ ],+ )+ result = apply(df, transform)+ assert_frame_equal(result, expected)++ @pytest.mark.xfail(+ reason="Casting both sides to Float64 causes precision loss at ~15 decimal places."+ )+ @staticmethod+ @pytest.mark.parametrize(+ "df",+ create_test_dataframes(+ {+ "A": [+ Decimal("1.12345678901234567"),+ Decimal("1.12345678901234568"),+ ]+ },+ exclude=["pandas"],+ ),+ )+ def test_filter_rows_in_decimal_precision_loss(+ df: DataFrameType,+ ) -> None:+ transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="A",+ operator="in",+ value=["1.12345678901234567"],+ ),+ ],+ )+ result = apply(df, transform)+ assert df_size(result) == 1++ @staticmethod+ @pytest.mark.parametrize(+ "df",+ create_test_dataframes(+ {+ "A": [+ Decimal("1.99"),+ Decimal("0.50"),+ Decimal("3.25"),+ ]+ },+ exclude=["pandas"],+ ),+ )+ def test_filter_rows_decimal_comparison_operators(+ df: DataFrameType,+ ) -> None:+ """Test ==, >=, <, not_in on Decimal columns with string values."""+ eq_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator="==", value="0.50")],+ )+ assert df_size(apply(df, eq_transform)) == 1++ gte_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator=">=", value="1.99")],+ )+ assert df_size(apply(df, gte_transform)) == 2++ lt_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator="<", value="1.00")],+ )+ assert df_size(apply(df, lt_transform)) == 1++ not_in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="A",+ operator="not_in",+ value=["0.50"],+ )+ ],+ )+ assert df_size(apply(df, not_in_transform)) == 2++ @staticmethod+ def test_filter_rows_in_decimal_pandas() -> None:+ """Pandas stores Decimal as object dtype; filters should still work."""+ df = pd.DataFrame(+ {+ "A": [Decimal("1.99"), Decimal("0.50"), Decimal("3.25")],+ }+ )++ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(column_id="A", operator="in", value=["1.99", "3.25"])+ ],+ )+ result = apply(df, in_transform)+ assert df_size(result) == 2++ eq_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator="==", value="0.50")],+ )+ assert df_size(apply(df, eq_transform)) == 1++ gte_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[Condition(column_id="A", operator=">=", value="1.99")],+ )+ assert df_size(apply(df, gte_transform)) == 2++ @staticmethod+ def test_filter_rows_date_pandas_object_dtype() -> None:+ """Pandas stores raw Python date objects as Object dtype.++ The Object handler must sample correctly via a LazyFrame+ (apply_transforms_to_df calls make_lazy) to detect the date type.+ """+ df = pd.DataFrame(+ {"d": [date(2024, 1, 1), date(2024, 6, 15), date(2024, 12, 31)]}+ )+ # Confirm pandas stored these as object, not datetime64+ assert df["d"].dtype == object++ eq_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(column_id="d", operator="==", value="2024-06-15")+ ],+ )+ assert df_size(apply(df, eq_transform)) == 1++ in_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(+ column_id="d",+ operator="in",+ value=["2024-01-01", "2024-12-31"],+ )+ ],+ )+ assert df_size(apply(df, in_transform)) == 2++ @staticmethod+ def test_filter_rows_date_pandas_object_dtype_with_leading_nulls() -> None:+ """Type detection should work even when the first rows are null."""+ df = pd.DataFrame(+ {"d": [None, None, date(2024, 1, 1), date(2024, 6, 15)]}+ )+ assert df["d"].dtype == object++ eq_transform = FilterRowsTransform(+ type=TransformType.FILTER_ROWS,+ operation="keep_rows",+ where=[+ Condition(column_id="d", operator="==", value="2024-06-15")+ ],+ )+ assert df_size(apply(df, eq_transform)) == 1+@staticmethod@pytest.mark.parametrize(("df", "expected"),