Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Convert decimals to float64 for filtering in tables (#8706)

Shahmir Varqha8eec31fe64ca
diff --git a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
index 5d9f63275a..5b53b12d26 100644
--- a/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
+++ b/marimo/_plugins/ui/_impl/dataframes/transforms/handlers.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import datetime
+import decimal
from functools import reduce
from typing import TYPE_CHECKING, Any, Callable
@@ -158,11 +159,12 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
elif dtype == nw.Time:
value = convert_value(value, datetime.time.fromisoformat)
elif dtype == nw.Object:
- # Object dtype may contain date/datetime values
- # (e.g., pandas with Python date objects)
+ # Object dtype may contain date/datetime/Decimal values
+ # (e.g., pandas stores these as Python objects)
try:
sample = (
- df.select(col(column_name).drop_nulls())
+ df.select(column_name)
+ .filter(~col(column_name).is_null())
.head(1)
.collect()
.get_column(column_name)
@@ -181,6 +183,10 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
value = convert_value(
value, datetime.time.fromisoformat
)
+ elif isinstance(sample[0], decimal.Decimal):
+ # Cast to Float64 so Decimal values can be compared, minor precision loss
+ value = convert_value(value, float)
+ column = column.cast(nw.Float64)
except Exception:
pass
@@ -195,6 +201,10 @@ class NarwhalsTransformHandler(TransformHandler[DataFrame]):
and dtype.is_float() # Note: this doesn't cover Object types for pandas
):
value = convert_value(value, float)
+ elif dtype is not None and dtype.is_decimal():
+ # Cast to Float64 so Decimal values can be compared, minor precision loss
+ value = convert_value(value, float)
+ column = column.cast(nw.Float64)
# Build the expression based on the operator
condition_expr: nw.Expr
diff --git a/marimo/_smoke_tests/tables/decimals.py b/marimo/_smoke_tests/tables/decimals.py
index c54776a996..66c3221ec3 100644
--- a/marimo/_smoke_tests/tables/decimals.py
+++ b/marimo/_smoke_tests/tables/decimals.py
@@ -1,36 +1,62 @@
import marimo
-__generated_with = "0.18.2"
+__generated_with = "0.20.4"
app = marimo.App(width="medium")
@app.cell
def _():
import marimo as mo
- return
+ from decimal import Decimal
+
+ return Decimal, mo
@app.cell
-def _():
+def _(Decimal, mo, pd):
import polars as pl
- decimal_data = pl.DataFrame(
- {"decimals": pl.Series(range(201), dtype=pl.Decimal(scale=2))}
+ _data = {
+ "product": ["Apple", "Banana", "Cherry", "Date"],
+ "price": pl.Series(
+ [
+ Decimal("1.99"),
+ Decimal("0.50"),
+ Decimal("3.25"),
+ Decimal("0.00"),
+ ],
+ dtype=pl.Decimal(18, 2),
+ ),
+ "tax": pl.Series(
+ [
+ Decimal("1.123456789012345678"),
+ Decimal("2.987654321098765432"),
+ Decimal("1.123456789012345678"),
+ Decimal("-3.492354321098723632"),
+ ],
+ dtype=pl.Decimal(36, 18),
+ ),
+ "quantity": [10, 5, 8, 3],
+ }
+
+ df = pl.DataFrame(_data)
+ pl_table = mo.ui.table(df, label="Polars table with decimal values")
+ pd_table = mo.ui.table(
+ pd.DataFrame(_data), label="Pandas table with decimal values"
)
- decimal_data
+ mo.vstack([pl_table, pd_table])
return
@app.cell
-def _():
+def _(Decimal):
import pandas as pd
- from decimal import Decimal
pandas_decimal_data = pd.DataFrame(
{"decimals": [Decimal(i) / Decimal("100") for i in range(201)]}
)
pandas_decimal_data
- return
+ return (pd,)
if __name__ == "__main__":
diff --git a/tests/_plugins/ui/_impl/dataframes/test_handlers.py b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
index 0a696f749c..56a5f1cc1d 100644
--- a/tests/_plugins/ui/_impl/dataframes/test_handlers.py
+++ b/tests/_plugins/ui/_impl/dataframes/test_handlers.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from datetime import date
+from decimal import Decimal
from typing import Any, Optional, cast
import narwhals.stable.v2 as nw
@@ -498,6 +499,249 @@ class TestTransformHandler:
result = apply(df, transform)
assert_frame_equal(result, expected)
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {"A": [Decimal("1.99"), Decimal("0.50"), Decimal("3.25")]},
+ exclude=["pandas"],
+ ),
+ create_test_dataframes(
+ {"A": [Decimal("1.99"), Decimal("3.25")]},
+ exclude=["pandas"],
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_in_decimal(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="A",
+ operator="in",
+ value=["1.99", "3.25"],
+ ),
+ ],
+ )
+ result = apply(df, transform)
+ assert_frame_equal(result, expected)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ ("df", "expected"),
+ list(
+ zip(
+ create_test_dataframes(
+ {
+ "A": [
+ Decimal("0.10"),
+ Decimal("0.20"),
+ Decimal("0.30"),
+ ]
+ },
+ exclude=["pandas"],
+ ),
+ create_test_dataframes(
+ {"A": [Decimal("0.10")]},
+ exclude=["pandas"],
+ ),
+ )
+ ),
+ )
+ def test_filter_rows_in_decimal_not_exactly_representable_as_float(
+ df: DataFrameType, expected: DataFrameType
+ ) -> None:
+ """0.1 is not exactly representable as a float, but filtering should still work."""
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="A",
+ operator="in",
+ value=["0.10"],
+ ),
+ ],
+ )
+ result = apply(df, transform)
+ assert_frame_equal(result, expected)
+
+ @pytest.mark.xfail(
+ reason="Casting both sides to Float64 causes precision loss at ~15 decimal places."
+ )
+ @staticmethod
+ @pytest.mark.parametrize(
+ "df",
+ create_test_dataframes(
+ {
+ "A": [
+ Decimal("1.12345678901234567"),
+ Decimal("1.12345678901234568"),
+ ]
+ },
+ exclude=["pandas"],
+ ),
+ )
+ def test_filter_rows_in_decimal_precision_loss(
+ df: DataFrameType,
+ ) -> None:
+ transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="A",
+ operator="in",
+ value=["1.12345678901234567"],
+ ),
+ ],
+ )
+ result = apply(df, transform)
+ assert df_size(result) == 1
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ "df",
+ create_test_dataframes(
+ {
+ "A": [
+ Decimal("1.99"),
+ Decimal("0.50"),
+ Decimal("3.25"),
+ ]
+ },
+ exclude=["pandas"],
+ ),
+ )
+ def test_filter_rows_decimal_comparison_operators(
+ df: DataFrameType,
+ ) -> None:
+ """Test ==, >=, <, not_in on Decimal columns with string values."""
+ eq_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator="==", value="0.50")],
+ )
+ assert df_size(apply(df, eq_transform)) == 1
+
+ gte_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator=">=", value="1.99")],
+ )
+ assert df_size(apply(df, gte_transform)) == 2
+
+ lt_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator="<", value="1.00")],
+ )
+ assert df_size(apply(df, lt_transform)) == 1
+
+ not_in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="A",
+ operator="not_in",
+ value=["0.50"],
+ )
+ ],
+ )
+ assert df_size(apply(df, not_in_transform)) == 2
+
+ @staticmethod
+ def test_filter_rows_in_decimal_pandas() -> None:
+ """Pandas stores Decimal as object dtype; filters should still work."""
+ df = pd.DataFrame(
+ {
+ "A": [Decimal("1.99"), Decimal("0.50"), Decimal("3.25")],
+ }
+ )
+
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(column_id="A", operator="in", value=["1.99", "3.25"])
+ ],
+ )
+ result = apply(df, in_transform)
+ assert df_size(result) == 2
+
+ eq_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator="==", value="0.50")],
+ )
+ assert df_size(apply(df, eq_transform)) == 1
+
+ gte_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[Condition(column_id="A", operator=">=", value="1.99")],
+ )
+ assert df_size(apply(df, gte_transform)) == 2
+
+ @staticmethod
+ def test_filter_rows_date_pandas_object_dtype() -> None:
+ """Pandas stores raw Python date objects as Object dtype.
+
+ The Object handler must sample correctly via a LazyFrame
+ (apply_transforms_to_df calls make_lazy) to detect the date type.
+ """
+ df = pd.DataFrame(
+ {"d": [date(2024, 1, 1), date(2024, 6, 15), date(2024, 12, 31)]}
+ )
+ # Confirm pandas stored these as object, not datetime64
+ assert df["d"].dtype == object
+
+ eq_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(column_id="d", operator="==", value="2024-06-15")
+ ],
+ )
+ assert df_size(apply(df, eq_transform)) == 1
+
+ in_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(
+ column_id="d",
+ operator="in",
+ value=["2024-01-01", "2024-12-31"],
+ )
+ ],
+ )
+ assert df_size(apply(df, in_transform)) == 2
+
+ @staticmethod
+ def test_filter_rows_date_pandas_object_dtype_with_leading_nulls() -> None:
+ """Type detection should work even when the first rows are null."""
+ df = pd.DataFrame(
+ {"d": [None, None, date(2024, 1, 1), date(2024, 6, 15)]}
+ )
+ assert df["d"].dtype == object
+
+ eq_transform = FilterRowsTransform(
+ type=TransformType.FILTER_ROWS,
+ operation="keep_rows",
+ where=[
+ Condition(column_id="d", operator="==", value="2024-06-15")
+ ],
+ )
+ assert df_size(apply(df, eq_transform)) == 1
+
@staticmethod
@pytest.mark.parametrize(
("df", "expected"),