Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

55.6 KB
Raw Blame
from __future__ import annotations

import ast
import datetime
import string
from typing import TYPE_CHECKING, Literal, cast

import narwhals.stable.v2 as nw
import pytest
from hypothesis import assume, example, given, settings, strategies as st

from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.dataframes.transforms.apply import (
    _apply_transforms,
)
from marimo._plugins.ui._impl.dataframes.transforms.handlers import (
    NarwhalsTransformHandler,
)
from marimo._plugins.ui._impl.dataframes.transforms.print_code import (
    python_print_ibis,
    python_print_pandas,
    python_print_polars,
    python_print_transforms,
)
from marimo._plugins.ui._impl.dataframes.transforms.types import (
    AggregateTransform,
    ColumnConversionTransform,
    ExpandDictTransform,
    ExplodeColumnsTransform,
    FilterCondition,
    FilterGroup,
    FilterRowsTransform,
    GroupByTransform,
    Operator,
    PivotTransform,
    RenameColumnTransform,
    SampleRowsTransform,
    SelectColumnsTransform,
    ShuffleRowsTransform,
    SortColumnTransform,
    Transform,
    Transformations,
    TransformType,
)

if TYPE_CHECKING:
    import pandas as pd
    import polars as pl

any_column_id = st.one_of(
    st.text(
        min_size=1,
        alphabet=string.ascii_letters + string.digits + "_",
    ),
)

defined_column_id = st.sampled_from(
    [
        "strings",
        "integers",
        "floats",
        "booleans",
        "dates",
        "times",
        "datetimes",
        "mixed",
        "dicts",
        "lists",
    ]
)


def create_transform_strategy(
    column_id: st.SearchStrategy[str],
    string_column_id: st.SearchStrategy[str | int] | None = None,
    bool_column_id: st.SearchStrategy[str | int] | None = None,
    comparison_column_id: st.SearchStrategy[str | int] | None = None,
    list_column_id: st.SearchStrategy[str | int] | None = None,
    df_size: int = 3,
) -> st.SearchStrategy[Transform]:
    column_ids = st.lists(column_id, min_size=1, unique=True)
    aggregation_column_ids = st.one_of(
        st.just([]),
        st.lists(column_id, min_size=1, unique=True),
    )
    index_column_ids = st.one_of(
        st.just([]),
        st.lists(column_id, min_size=1, unique=True),
    )

    if string_column_id is None:
        string_column_id = column_id
    if bool_column_id is None:
        bool_column_id = column_id
    if comparison_column_id is None:
        comparison_column_id = column_id
    if list_column_id is None:
        list_column_id = column_id

    numpy_data_type = st.sampled_from(["str", "bool"])
    aggregation = st.sampled_from(
        ["count", "sum", "mean", "median", "min", "max"]
    )

    # Strategies for each condition type
    comparison_condition_strategy = st.builds(
        FilterCondition,
        type=st.just("condition"),
        column_id=comparison_column_id,
        operator=st.sampled_from(["==", "!=", "<", ">", "<=", ">="]),
        value=st.one_of(
            st.integers(), st.floats(allow_infinity=False, allow_nan=False)
        ),
    )

    boolean_condition_strategy = st.builds(
        FilterCondition,
        type=st.just("condition"),
        column_id=bool_column_id,
        operator=st.sampled_from(["is_true", "is_false"]),
        value=st.just(None),
    )

    string_condition_strategy = st.builds(
        FilterCondition,
        type=st.just("condition"),
        column_id=string_column_id,
        operator=st.sampled_from(
            [
                "equals",
                "does_not_equal",
                "contains",
                "regex",
                "starts_with",
                "ends_with",
            ]
        ),
        value=st.text(alphabet=string.ascii_letters + string.digits),
    )

    list_condition_strategy = st.builds(
        FilterCondition,
        type=st.just("condition"),
        column_id=list_column_id,
        operator=st.sampled_from(["in", "not_in"]),
        value=st.lists(st.one_of(st.text()), min_size=1),
    )

    condition_strategy = st.one_of(
        comparison_condition_strategy,
        boolean_condition_strategy,
        string_condition_strategy,
        list_condition_strategy,
    )

    column_conversion_transform_strategy = st.builds(
        ColumnConversionTransform,
        type=st.just(TransformType.COLUMN_CONVERSION),
        column_id=column_id,
        data_type=numpy_data_type,
        errors=st.sampled_from(["ignore", "raise"]),
    )

    rename_column_transform_strategy = st.builds(
        RenameColumnTransform,
        type=st.just(TransformType.RENAME_COLUMN),
        column_id=column_id,
        new_column_id=column_id,
    ).filter(lambda x: x.column_id != x.new_column_id)

    sort_column_transform_strategy = st.builds(
        SortColumnTransform,
        type=st.just(TransformType.SORT_COLUMN),
        column_id=column_id,
        ascending=st.booleans(),
        na_position=st.sampled_from(["first", "last"]),
    )

    filter_group_strategy = st.builds(
        FilterGroup,
        type=st.just("group"),
        operator=st.just("and"),
        children=st.lists(condition_strategy, min_size=1),
    )

    filter_rows_transform_strategy = st.builds(
        FilterRowsTransform,
        type=st.just(TransformType.FILTER_ROWS),
        operation=st.sampled_from(["keep_rows", "remove_rows"]),
        where=filter_group_strategy,
    )

    group_by_transform_strategy = st.builds(
        GroupByTransform,
        type=st.just(TransformType.GROUP_BY),
        column_ids=column_ids,
        drop_na=st.booleans(),
        aggregation=aggregation,
        aggregation_column_ids=aggregation_column_ids,
    )

    aggregate_transform_strategy = st.builds(
        AggregateTransform,
        type=st.just(TransformType.AGGREGATE),
        column_ids=column_ids,
        aggregations=st.lists(aggregation, min_size=1),
    )

    select_columns_transform_strategy = st.builds(
        SelectColumnsTransform,
        type=st.just(TransformType.SELECT_COLUMNS),
        column_ids=column_ids,
    )

    shuffle_rows_transform_strategy = st.builds(
        ShuffleRowsTransform,
        type=st.just(TransformType.SHUFFLE_ROWS),
        seed=st.integers(),
    )

    sample_rows_transform_strategy = st.builds(
        SampleRowsTransform,
        type=st.just(TransformType.SAMPLE_ROWS),
        n=st.integers(min_value=1, max_value=df_size),
        replace=st.booleans(),
        seed=st.integers(),
    )

    explode_columns_transform_strategy = st.builds(
        ExplodeColumnsTransform,
        type=st.just(TransformType.EXPLODE_COLUMNS),
        column_ids=column_ids,
    )

    expand_dict_transform_strategy = st.builds(
        ExpandDictTransform,
        type=st.just(TransformType.EXPAND_DICT),
        column_id=column_id,
    )

    pivot_transform_strategy = st.builds(
        PivotTransform,
        type=st.just(TransformType.PIVOT),
        column_ids=column_ids,
        index_column_ids=index_column_ids,
        aggregation=aggregation,
        value_column_ids=aggregation_column_ids,
    )

    # Combine all transform strategies
    transform_strategy = st.one_of(
        column_conversion_transform_strategy,
        rename_column_transform_strategy,
        sort_column_transform_strategy,
        filter_rows_transform_strategy,
        group_by_transform_strategy,
        aggregate_transform_strategy,
        select_columns_transform_strategy,
        shuffle_rows_transform_strategy,
        sample_rows_transform_strategy,
        explode_columns_transform_strategy,
        expand_dict_transform_strategy,
        pivot_transform_strategy,
    )

    return transform_strategy


transformations_strategy = st.builds(
    Transformations,
    transforms=st.lists(create_transform_strategy(any_column_id), min_size=1),
)


def _assert_string_filter_matches_runtime(
    backend: str,
    strings: list[str | None],
    operator: Operator,
    value: str,
    negate: bool,
    operation: Literal["keep_rows", "remove_rows"],
) -> None:
    data = {"strings": strings, "row_id": list(range(len(strings)))}
    if backend.startswith("pandas"):
        pd = pytest.importorskip("pandas")
        df = pd.DataFrame(data).astype(
            {"strings": "object" if backend == "pandas-object" else "string"}
        )
        printer = python_print_pandas
        namespace = {"pd": pd, "df": df}
    elif backend == "polars":
        pl = pytest.importorskip("polars")
        df = pl.DataFrame(data, schema_overrides={"strings": pl.String})
        printer = python_print_polars
        namespace = {"pl": pl, "df": df}
    else:
        ibis = pytest.importorskip("ibis")
        pytest.importorskip("duckdb")
        df = ibis.memtable(
            data, schema={"strings": "string", "row_id": "int64"}
        )
        printer = python_print_ibis
        namespace = {"ibis": ibis, "df": df}

    transform = FilterRowsTransform(
        type=TransformType.FILTER_ROWS,
        operation=operation,
        where=FilterGroup(
            operator="and",
            children=[
                FilterCondition(
                    column_id="strings",
                    operator=operator,
                    value=value,
                    negate=negate,
                )
            ],
        ),
    )
    expected = _apply_transforms(
        nw.from_native(df).lazy(),
        NarwhalsTransformHandler(),
        Transformations([transform]),
    ).collect()
    code = python_print_transforms("df", list(data), [transform], printer)
    exec(code, namespace)
    actual = nw.from_native(namespace["df_next"]).lazy().collect()
    assert actual.to_dict(as_series=False) == expected.to_dict(as_series=False)


@pytest.mark.parametrize(
    "backend", ["pandas-object", "pandas-string", "polars", "ibis"]
)
@pytest.mark.parametrize(
    "operator", ["contains", "regex", "starts_with", "ends_with"]
)
@pytest.mark.parametrize("value", ["", "a", "a.c"])
@pytest.mark.parametrize("negate", [False, True])
@pytest.mark.parametrize("operation", ["keep_rows", "remove_rows"])
def test_print_code_string_filters_with_nulls(
    backend: str,
    operator: Operator,
    value: str,
    negate: bool,
    operation: Literal["keep_rows", "remove_rows"],
) -> None:
    _assert_string_filter_matches_runtime(
        backend,
        [None, "", "abc", "a.c", "cba", "zzz"],
        operator,
        value,
        negate,
        operation,
    )


@pytest.mark.parametrize(
    "backend", ["pandas-object", "pandas-string", "polars", "ibis"]
)
@given(
    strings=st.lists(
        st.one_of(st.none(), st.text(alphabet="abc .", max_size=10)),
        min_size=1,
        max_size=10,
    ),
    operator=st.sampled_from(
        ["contains", "regex", "starts_with", "ends_with"]
    ),
    value=st.text(alphabet="abc.", max_size=3),
    negate=st.booleans(),
    operation=st.sampled_from(["keep_rows", "remove_rows"]),
)
@settings(deadline=None)
def test_print_code_string_filters_with_generated_nulls(
    backend: str,
    strings: list[str | None],
    operator: Operator,
    value: str,
    negate: bool,
    operation: Literal["keep_rows", "remove_rows"],
) -> None:
    _assert_string_filter_matches_runtime(
        backend, strings, operator, value, negate, operation
    )


def _validate_code(code: str):
    try:
        ast.parse(code)
    except SyntaxError as e:
        raise SyntaxError(f"Invalid Python code for {code}") from e


@given(transform=create_transform_strategy(any_column_id))
@settings(deadline=None)
def test_python_print_pandas(transform: Transform):
    df_name = "df"
    result = python_print_pandas(df_name, ["a"], transform)
    assert isinstance(result, str)
    _validate_code(result)
    assert df_name in result


@given(transform=create_transform_strategy(any_column_id))
@settings(deadline=None)
def test_python_print_polars(transform: Transform):
    df_name = "df"
    result = python_print_polars(df_name, ["a"], transform)
    assert isinstance(result, str)
    _validate_code(result)
    assert df_name in result


@given(transformations=transformations_strategy)
@settings(deadline=None)
@pytest.mark.skip
def test_python_print_ibis(transformations: Transformations):
    df_name = "df"
    result = python_print_transforms(
        df_name, ["a"], transformations.transforms, python_print_ibis
    )
    assert isinstance(result, str)
    _validate_code(result)
    assert df_name in result


@given(transformations=transformations_strategy)
@settings(deadline=None)
def test_python_print_transforms(transformations: Transformations):
    df_name = "df"
    # Pandas
    result = python_print_transforms(
        df_name, ["a"], transformations.transforms, python_print_pandas
    )
    assert isinstance(result, str)
    _validate_code(result)
    assert df_name in result
    # Polars
    result = python_print_transforms(
        df_name, ["a"], transformations.transforms, python_print_polars
    )
    assert isinstance(result, str)
    _validate_code(result)
    assert df_name in result
    # Ibis
    # TODO: test ibis python print
    # result = python_print_transforms(
    #     df_name, ["a"], transformations.transforms, python_print_ibis
    # )
    # assert isinstance(result, str)
    # _validate_code(result)
    # assert df_name in result


@given(
    transform=create_transform_strategy(
        defined_column_id,
        string_column_id=st.just("strings"),
        bool_column_id=st.just("booleans"),
        comparison_column_id=st.sampled_from(
            ["integers", "floats", "dates", "times", "datetimes"]
        ),
        list_column_id=st.just("strings"),
        df_size=3,
    )
)
@settings(deadline=None)
@pytest.mark.skipif(
    not DependencyManager.pandas.has(), reason="pandas not installed"
)
def test_print_code_result_matches_actual_transform_pandas(
    transform: Transform,
):
    import pandas as pd

    transformations = Transformations([transform])

    my_df = pd.DataFrame(
        {
            "strings": ["a", "b", "c"],
            "integers": [1, 2, 3],
            "floats": [1.0, 2.0, 3.0],
            "booleans": [True, False, True],
            "dates": [pd.Timestamp("2021-01-01")] * 3,
            "times": [pd.Timestamp("2021-01-01 00:00:00")] * 3,
            "datetimes": [pd.Timestamp("2021-01-01 00:00:00")] * 3,
            "mixed": [1, 2.0, "3"],
            "dicts": [{"a": 1}, {"b": 2}, {"c": 3}],
            "lists": [[1, 2], [3, 4], [5, 6]],
        }
    )

    # Exclude shuffle and sample rows
    assume(
        transform.type
        not in {TransformType.SHUFFLE_ROWS, TransformType.SAMPLE_ROWS}
    )
    # Ignore date, time, datetime columns in filter rows
    if transform.type == TransformType.FILTER_ROWS:
        assume(
            not any(
                condition.column_id in {"dates", "times", "datetimes"}
                for condition in transform.where.children
            )
        )
    # Ignore groupby mean
    if transform.type == TransformType.GROUP_BY:
        assume(transform.aggregation != "mean")
        assume("dict" not in transform.aggregation_column_ids)

    if transform.type == TransformType.PIVOT:
        assume(
            (
                set(transform.column_ids) & set(transform.index_column_ids)
                == set()
            )
            and (
                set(transform.column_ids) & set(transform.value_column_ids)
                == set()
            )
            and (
                set(transform.index_column_ids)
                & set(transform.value_column_ids)
                == set()
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.index_column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.value_column_ids
            )
        )

    # Pandas
    pandas_code = python_print_transforms(
        "my_df",
        list(my_df.columns),
        transformations.transforms,
        python_print_pandas,
    )
    assert pandas_code

    try:
        loc = {"pd": pd, "my_df": my_df.copy()}
        exec(pandas_code, {}, loc)
        code_result = loc.get("my_df_next")
    except Exception as code_error:
        code_result = code_error

    try:
        nw_df = nw.from_native(my_df.copy(), eager_only=True).lazy()
        result_nw = _apply_transforms(
            nw_df,
            NarwhalsTransformHandler(),
            transformations,
        )
        real_result = result_nw.collect().to_native()
    except Exception as real_error:
        real_result = real_error

    if isinstance(code_result, Exception) or isinstance(
        real_result, Exception
    ):
        # Allow different error types between pandas and narwhals
        import narwhals.exceptions as nw_exc

        if isinstance(real_result, nw_exc.DuplicateError):
            # Pandas doesn't raise DuplicateError, it just creates duplicate columns
            # So if narwhals raised DuplicateError, it's expected that pandas succeeded
            assert not isinstance(code_result, Exception)
            return
        if isinstance(real_result, nw_exc.InvalidOperationError):
            # Pandas may allow some operations that narwhals doesn't
            # If narwhals raised InvalidOperationError, pandas might have succeeded
            # or raised a different error - just skip comparison
            return
        if isinstance(real_result, (AttributeError, TypeError)):
            # Narwhals may raise AttributeError for some operations (e.g., duplicate column names in aggregate)
            # Pandas might have succeeded - just skip comparison
            return
        assert type(code_result) is type(real_result)
        assert str(code_result) == str(real_result)
    else:
        # If series, convert to dataframe
        if isinstance(code_result, pd.Series):
            code_result = code_result.to_frame()
        if isinstance(real_result, pd.Series):
            real_result = real_result.to_frame()

        # For pivot transform the column order can be different, enforce column order by sorting.
        if transform.type == TransformType.PIVOT:
            code_result = code_result.loc[:, sorted(code_result.columns)]
            real_result = real_result.loc[:, sorted(real_result.columns)]

        code_result = cast(pd.DataFrame, code_result).reset_index(drop=True)
        real_result = real_result.reset_index(drop=True)

        # For group_by transforms, the row order might differ
        # Sort both dataframes by all columns before comparing
        if transform.type == TransformType.GROUP_BY:
            # Filter out columns with unhashable types (dicts, lists)
            sortable_cols = [
                col
                for col in code_result.columns
                if not any(
                    isinstance(val, (dict, list))
                    for val in code_result[col]
                    if val is not None
                )
            ]
            if sortable_cols:
                code_result = code_result.sort_values(
                    by=sortable_cols,
                    key=lambda col: col.astype(str),
                ).reset_index(drop=True)
                real_result = real_result.sort_values(
                    by=sortable_cols,
                    key=lambda col: col.astype(str),
                ).reset_index(drop=True)

        # Pivot can produce dtype differences between native pandas
        # (object) and narwhals (StringDtype) for string-valued columns.
        check_dtype = transform.type != TransformType.PIVOT
        pd.testing.assert_frame_equal(
            code_result, real_result, check_dtype=check_dtype
        )


@pytest.mark.skipif(
    not DependencyManager.pandas.has(), reason="pandas not installed"
)
def test_print_code_expand_dict_nested_dict_pandas() -> None:
    import pandas as pd

    transform = ExpandDictTransform(
        type=TransformType.EXPAND_DICT,
        column_id="dicts",
    )
    transformations = Transformations([transform])
    my_df = pd.DataFrame(
        {
            "dicts": [{"a": 1, "nested": {"x": 2}}, None],
            "other": [10, 20],
        }
    )

    pandas_code = python_print_transforms(
        "my_df",
        list(my_df.columns),
        transformations.transforms,
        python_print_pandas,
    )
    assert pandas_code

    loc = {"pd": pd, "my_df": my_df.copy()}
    exec(pandas_code, {}, loc)
    code_result = loc["my_df_next"]

    nw_df = nw.from_native(my_df.copy(), eager_only=True).lazy()
    result_nw = _apply_transforms(
        nw_df,
        NarwhalsTransformHandler(),
        transformations,
    )
    real_result = result_nw.collect().to_native().reset_index(drop=True)

    pd.testing.assert_frame_equal(
        code_result.reset_index(drop=True),
        real_result,
    )

    assert list(code_result.columns) == ["a", "nested", "other"]


@pytest.mark.skipif(
    not DependencyManager.pandas.has(), reason="pandas not installed"
)
def test_print_code_expand_dict_nan_pandas() -> None:
    import pandas as pd

    transform = ExpandDictTransform(
        type=TransformType.EXPAND_DICT,
        column_id="dicts",
    )
    transformations = Transformations([transform])
    my_df = pd.DataFrame(
        {
            "dicts": [{"a": 1}, float("nan"), {"a": 3}],
            "other": [10, 20, 30],
        }
    )

    pandas_code = python_print_transforms(
        "my_df",
        list(my_df.columns),
        transformations.transforms,
        python_print_pandas,
    )
    assert pandas_code

    loc = {"pd": pd, "my_df": my_df.copy()}
    exec(pandas_code, {}, loc)
    code_result = loc["my_df_next"]

    nw_df = nw.from_native(my_df.copy(), eager_only=True).lazy()
    result_nw = _apply_transforms(
        nw_df,
        NarwhalsTransformHandler(),
        transformations,
    )
    real_result = result_nw.collect().to_native().reset_index(drop=True)

    pd.testing.assert_frame_equal(
        code_result.reset_index(drop=True),
        real_result,
    )


# The values here are valid strings that are also regexes: "a.c" matches
# "abc" when interpreted as a regex, and "(" is not a valid regex at all. If
# `contains` is emitted without a literal flag, the generated pandas code
# diverges from the widget (or raises), so this test fails.
@pytest.mark.parametrize("value", ["a.c", "("])
@pytest.mark.skipif(
    not DependencyManager.pandas.has(), reason="pandas not installed"
)
def test_print_code_contains_is_literal_pandas(value: str) -> None:
    import pandas as pd

    my_df = pd.DataFrame({"strings": ["abc", "a.c", "f(x)", "zzz"]})

    transformations = Transformations(
        [
            FilterRowsTransform(
                type=TransformType.FILTER_ROWS,
                operation="keep_rows",
                where=FilterGroup(
                    type="group",
                    operator="and",
                    children=[
                        FilterCondition(
                            type="condition",
                            column_id="strings",
                            operator="contains",
                            value=value,
                        )
                    ],
                ),
            )
        ]
    )

    pandas_code = python_print_transforms(
        "my_df",
        list(my_df.columns),
        transformations.transforms,
        python_print_pandas,
    )

    loc = {"pd": pd, "my_df": my_df.copy()}
    exec(pandas_code, {}, loc)
    code_result = loc["my_df_next"]

    nw_df = nw.from_native(my_df.copy(), eager_only=True).lazy()
    real_result = (
        _apply_transforms(
            nw_df,
            NarwhalsTransformHandler(),
            transformations,
        )
        .collect()
        .to_native()
        .reset_index(drop=True)
    )

    pd.testing.assert_frame_equal(
        code_result.reset_index(drop=True),
        real_result,
    )


@given(
    transform=create_transform_strategy(
        defined_column_id,
        string_column_id=st.just("strings"),
        bool_column_id=st.just("booleans"),
        comparison_column_id=st.sampled_from(
            ["integers", "floats", "dates", "times", "datetimes"]
        ),
        list_column_id=st.just("strings"),
        df_size=3,
    )
)
@example(
    transform=SortColumnTransform(
        type=TransformType.SORT_COLUMN,
        column_id="dates",
        ascending=False,
        na_position="first",
    )
)
@example(
    transform=PivotTransform(
        type=TransformType.PIVOT,
        column_ids=["strings"],
        index_column_ids=["integers"],
        value_column_ids=["floats"],
        aggregation="median",
    )
)
@settings(deadline=None)
@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
def test_print_code_result_matches_actual_transform_polars(
    transform: Transform,
):
    import polars as pl
    import polars.testing as pl_testing

    transformations = Transformations([transform])

    my_df = pl.DataFrame(
        {
            "strings": ["a", "b", "c"],
            "integers": [1, 2, 3],
            "floats": [1.0, 2.0, 3.0],
            "booleans": [True, False, True],
            "dates": [datetime.date(2021, 1, 1)] * 3,
            "times": [datetime.time(0, 0, 0)] * 3,
            "datetimes": [datetime.datetime(2021, 1, 1)] * 3,
            "mixed": [1, 2, 3],
            "dicts": [{"a": 1}, {"b": 2}, {"c": 3}],
            "lists": [[1, 2], [3, 4], [5, 6]],
        },
    )

    # Exclude shuffle and sample rows
    assume(
        transform.type
        not in {TransformType.SHUFFLE_ROWS, TransformType.SAMPLE_ROWS}
    )

    # Ignore date, time, datetime columns in filter rows
    if transform.type == TransformType.FILTER_ROWS:
        assume(
            not any(
                condition.column_id in {"dates", "times", "datetimes"}
                for condition in transform.where.children
            )
        )
    # Only explode columns for lists
    if transform.type == TransformType.EXPLODE_COLUMNS:
        assume(all(column_id == "lists" for column_id in transform.column_ids))
    # Only expand dict for dicts
    if transform.type == TransformType.EXPAND_DICT:
        assume("dicts" == transform.column_id)
    # Don't sort on dicts or lists
    if transform.type == TransformType.SORT_COLUMN:
        assume(transform.column_id not in {"dicts", "lists"})
    # Skip aggregation
    # TODO: unimplemented
    if transform.type == TransformType.AGGREGATE:
        assume(False)
    # Don't group by unhashable types (lists, dicts) as ordering is non-deterministic
    if transform.type == TransformType.GROUP_BY:
        assume(
            not any(
                column_id in {"lists", "dicts"}
                for column_id in transform.column_ids
            )
        )
        # When aggregation_column_ids is empty, all remaining columns become
        # aggregation columns. sum/mean fail on non-numeric columns (strings, etc.)
        if not transform.aggregation_column_ids and transform.aggregation in {
            "sum",
            "mean",
        }:
            assume(False)
        # Polars does not support sum/mean/median on string or other non-numeric columns
        if transform.aggregation in {"sum", "mean", "median"}:
            if not transform.aggregation_column_ids:
                assume(False)  # all columns used; would include strings
            else:
                numeric_only = {"integers", "floats", "mixed"}
                assume(
                    all(
                        col in numeric_only
                        for col in transform.aggregation_column_ids
                    )
                )
    if transform.type == TransformType.PIVOT:
        assume(
            (
                set(transform.column_ids) & set(transform.index_column_ids)
                == set()
            )
            and (
                set(transform.column_ids) & set(transform.value_column_ids)
                == set()
            )
            and (
                set(transform.index_column_ids)
                & set(transform.value_column_ids)
                == set()
            )
        )
        # Exclude lists, dicts, datetimes from pivot columns
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.index_column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.value_column_ids
            )
        )
        # Exclude booleans from pivot column_ids - polars uses lowercase (true/false)
        # while narwhals uses Python's str() which gives True/False
        assume(
            not any(
                column_id == "booleans" for column_id in transform.column_ids
            )
        )
        # When value_column_ids is empty, all remaining columns become values.
        # sum/mean fail on non-numeric columns; median on lists can panic
        # in Polars' streaming engine.
        if not transform.value_column_ids and transform.aggregation in {
            "sum",
            "mean",
            "median",
        }:
            assume(False)
        # Polars does not support sum/mean on string or other non-numeric columns
        if (
            transform.aggregation in {"sum", "mean"}
            and transform.value_column_ids
        ):
            numeric_only = {"integers", "floats", "mixed"}
            assume(
                not any(
                    col not in numeric_only
                    for col in transform.value_column_ids
                )
            )

    # Polars
    polars_code = python_print_transforms(
        "my_df", my_df.columns, transformations.transforms, python_print_polars
    )
    assert polars_code

    try:
        loc = {"pl": pl, "my_df": my_df.clone()}
        exec(polars_code, globals(), loc)
        code_result = loc.get("my_df_next")
    except Exception as code_error:
        code_result = code_error

    try:
        nw_df = nw.from_native(my_df.clone(), eager_only=True).lazy()
        result_nw = _apply_transforms(
            nw_df,
            NarwhalsTransformHandler(),
            transformations,
        )
        real_result = result_nw.collect().to_native()
    except Exception as real_error:
        real_result = real_error

    if isinstance(code_result, Exception) or isinstance(
        real_result, Exception
    ):
        # Allow different error types between polars and narwhals
        import narwhals.exceptions as nw_exc
        import polars.exceptions as pl_exc

        if isinstance(real_result, nw_exc.DuplicateError) and isinstance(
            code_result, pl_exc.DuplicateError
        ):
            # Both raised duplicate errors, just different types - this is OK
            return
        if isinstance(real_result, nw_exc.DuplicateError):
            # Polars raised DuplicateError from the generated code
            # but narwhals also raised DuplicateError - this is OK
            assert isinstance(code_result, pl_exc.DuplicateError)
            return
        if isinstance(
            real_result, nw_exc.InvalidOperationError
        ) and isinstance(code_result, pl_exc.InvalidOperationError):
            # Both raised invalid operation errors, just different types - this is OK
            return
        if isinstance(real_result, nw_exc.InvalidOperationError):
            # Polars may allow some operations that narwhals doesn't
            # If narwhals raised InvalidOperationError, just skip comparison
            return
        assert type(code_result) is type(real_result)
        assert str(code_result) == str(real_result)
    else:
        # If series, convert to dataframe
        if isinstance(code_result, pl.Series):
            code_result = code_result.to_frame()
        if isinstance(real_result, pl.Series):
            real_result = real_result.to_frame()

        # For pivot transform the column order can be different, enforce column order by sorting.
        if transform.type == TransformType.PIVOT:
            code_result = code_result.select(
                *sorted(code_result.collect_schema().names())
            )
            real_result = real_result.select(
                *sorted(real_result.collect_schema().names())
            )

        code_result = cast(pl.DataFrame, code_result)
        # Compare column names
        assert code_result.columns == real_result.columns

        # For group_by transforms, the row order might differ even with maintain_order=True
        # Sort both dataframes by all columns before comparing
        # Note: We exclude grouping by unhashable types (lists, dicts) via assume() above,
        # so all columns should be sortable
        if transform.type == TransformType.GROUP_BY:
            import polars.datatypes as pl_dtypes

            sortable_cols = [
                col
                for col in code_result.columns
                if code_result[col].dtype
                not in (pl_dtypes.List, pl_dtypes.Struct, pl_dtypes.Object)
            ]
            if sortable_cols:
                code_result = code_result.sort(sortable_cols)
                real_result = real_result.sort(sortable_cols)

        if transform.type == TransformType.SORT_COLUMN:
            # Ties may differ between eager and lazy sorts. Check key order
            # before comparing rows using the fixture's unique string column.
            pl_testing.assert_series_equal(
                code_result[transform.column_id],
                real_result[transform.column_id],
            )
            code_result = code_result.sort("strings")
            real_result = real_result.sort("strings")

        pl_testing.assert_frame_equal(code_result, real_result)


@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
@pytest.mark.parametrize("lazy", [False, True])
@pytest.mark.parametrize("errors", ["raise", "ignore"])
@pytest.mark.parametrize(
    ("data_type", "values", "expected_values"),
    [
        (
            "date",
            ["2026-10-07", None, "2026-10-08"],
            [datetime.date(2026, 10, 7), None, datetime.date(2026, 10, 8)],
        ),
        (
            "datetime64",
            ["2026-10-07 12:34:56.123456", None, "2026-10-08 00:00:00"],
            [
                datetime.datetime(2026, 10, 7, 12, 34, 56, 123456),
                None,
                datetime.datetime(2026, 10, 8),
            ],
        ),
        (
            "datetime64",
            [datetime.date(2026, 10, 7), None],
            [datetime.datetime(2026, 10, 7), None],
        ),
        (
            "date",
            [datetime.datetime(2026, 10, 7, 12), None],
            [datetime.date(2026, 10, 7), None],
        ),
        (
            "datetime64",
            [0, 1_000_000],
            [
                datetime.datetime(1970, 1, 1),
                datetime.datetime(1970, 1, 1, 0, 0, 1),
            ],
        ),
    ],
)
def test_polars_temporal_conversion_code_matches_runtime(
    lazy: bool,
    errors: Literal["raise", "ignore"],
    data_type: str,
    values: list[str | datetime.date | int | None],
    expected_values: list[datetime.date | None],
) -> None:
    import polars as pl
    import polars.testing as pl_testing

    column = 'event "time"'
    original = pl.DataFrame({column: values, "id": range(len(values))})
    source = original.lazy() if lazy else original
    transform = ColumnConversionTransform(
        type=TransformType.COLUMN_CONVERSION,
        column_id=column,
        data_type=data_type,
        errors=errors,
    )
    runtime = (
        NarwhalsTransformHandler.handle_column_conversion(
            nw.from_native(source).lazy(), transform
        )
        .collect()
        .to_native()
    )
    namespace = {"pl": pl, "df": source}
    exec(
        python_print_transforms(
            "df", original.columns, [transform], python_print_polars
        ),
        namespace,
    )
    generated = namespace["df_next"]
    if lazy:
        generated = generated.collect()
    expected = pl.DataFrame(
        {column: expected_values, "id": range(len(values))}
    )
    pl_testing.assert_frame_equal(runtime, expected)
    pl_testing.assert_frame_equal(generated, expected)
    pl_testing.assert_frame_equal(
        original, pl.DataFrame({column: values, "id": range(len(values))})
    )


@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
@pytest.mark.parametrize("lazy", [False, True])
@pytest.mark.parametrize("multiple_columns", [False, True])
@pytest.mark.parametrize("empty_frame", [False, True])
def test_polars_explode_code_preserves_empty_and_null_rows(
    lazy: bool, multiple_columns: bool, empty_frame: bool
) -> None:
    import polars as pl
    import polars.testing as pl_testing

    column = 'list "values"'
    original = pl.DataFrame(
        {
            column: [[1, 2], [], None],
            "other": [[3, 4], [], None],
            "id": [0, 1, 2],
        }
    )
    columns = [column, "other"] if multiple_columns else [column]
    expected = pl.DataFrame(
        {
            column: [1, 2, None, None],
            "other": (
                [3, 4, None, None]
                if multiple_columns
                else [[3, 4], [3, 4], [], None]
            ),
            "id": [0, 0, 1, 2],
        }
    )
    if empty_frame:
        original = original.head(0)
        expected = expected.head(0)
    source = original.lazy() if lazy else original
    transform = ExplodeColumnsTransform(
        type=TransformType.EXPLODE_COLUMNS, column_ids=columns
    )
    runtime = (
        NarwhalsTransformHandler.handle_explode_columns(
            nw.from_native(source).lazy(), transform
        )
        .collect()
        .to_native()
    )
    namespace = {"pl": pl, "df": source}
    exec(
        python_print_transforms(
            "df", original.columns, [transform], python_print_polars
        ),
        namespace,
    )
    generated = namespace["df_next"]
    if lazy:
        generated = generated.collect()
    pl_testing.assert_frame_equal(runtime, expected)
    pl_testing.assert_frame_equal(generated, expected)


@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
@pytest.mark.parametrize("values", [[], [None, None]])
@pytest.mark.parametrize("data_type", ["date", "datetime64"])
def test_polars_temporal_conversion_without_non_null_values(
    values: list[None], data_type: str
) -> None:
    import polars as pl
    import polars.testing as pl_testing

    source = pl.LazyFrame(
        {"event_time": values}, schema={"event_time": pl.String}
    )
    transform = ColumnConversionTransform(
        type=TransformType.COLUMN_CONVERSION,
        column_id="event_time",
        data_type=data_type,
        errors="raise",
    )
    runtime = (
        NarwhalsTransformHandler.handle_column_conversion(
            nw.from_native(source), transform
        )
        .collect()
        .to_native()
    )
    namespace = {"pl": pl, "df": source}
    exec(
        python_print_transforms(
            "df", ["event_time"], [transform], python_print_polars
        ),
        namespace,
    )
    dtype = pl.Date if data_type == "date" else pl.Datetime
    expected = pl.DataFrame(
        {"event_time": values}, schema={"event_time": dtype}
    )
    pl_testing.assert_frame_equal(runtime, expected)
    pl_testing.assert_frame_equal(namespace["df_next"].collect(), expected)


@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
@pytest.mark.parametrize("data_type", ["date", "datetime64"])
@pytest.mark.parametrize("errors", ["raise", "ignore"])
def test_polars_temporal_conversion_invalid_strings(
    data_type: str, errors: Literal["raise", "ignore"]
) -> None:
    import polars as pl

    source = pl.LazyFrame({"event_time": ["2026-10-07", "invalid", None]})
    transform = ColumnConversionTransform(
        type=TransformType.COLUMN_CONVERSION,
        column_id="event_time",
        data_type=data_type,
        errors=errors,
    )
    runtime = NarwhalsTransformHandler.handle_column_conversion(
        nw.from_native(source), transform
    )
    namespace = {"pl": pl, "df": source}
    exec(
        python_print_transforms(
            "df", ["event_time"], [transform], python_print_polars
        ),
        namespace,
    )
    generated = namespace["df_next"]
    with pytest.raises(nw.exceptions.InvalidOperationError):
        runtime.collect()
    with pytest.raises(pl.exceptions.InvalidOperationError):
        generated.collect()


@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
def test_print_code_expand_dict_nested_dict_polars() -> None:
    import polars as pl
    import polars.testing as pl_testing

    transform = ExpandDictTransform(
        type=TransformType.EXPAND_DICT,
        column_id="dicts",
    )
    transformations = Transformations([transform])
    my_df = pl.DataFrame(
        {
            "dicts": [{"a": 1, "nested": {"x": 2}}, None],
            "other": [10, 20],
        }
    )

    polars_code = python_print_transforms(
        "my_df",
        my_df.columns,
        transformations.transforms,
        python_print_polars,
    )
    assert polars_code

    loc = {"pl": pl, "my_df": my_df.clone()}
    exec(polars_code, globals(), loc)
    code_result = loc["my_df_next"]

    nw_df = nw.from_native(my_df.clone(), eager_only=True).lazy()
    result_nw = _apply_transforms(
        nw_df,
        NarwhalsTransformHandler(),
        transformations,
    )
    real_result = result_nw.collect().to_native()

    pl_testing.assert_frame_equal(code_result, real_result)

    assert code_result.columns == ["a", "nested", "other"]


# The values here are valid strings that are also regexes: "a.c" matches
# "abc" when interpreted as a regex, and "(" is not a valid regex at all. If
# `contains` is emitted without a literal flag, the generated polars code
# diverges from the widget (or raises), so this test fails.
@pytest.mark.parametrize("value", ["a.c", "("])
@pytest.mark.skipif(
    not DependencyManager.polars.has(), reason="polars not installed"
)
def test_print_code_contains_is_literal_polars(value: str) -> None:
    import polars as pl
    import polars.testing as pl_testing

    my_df = pl.DataFrame({"strings": ["abc", "a.c", "f(x)", "zzz"]})

    transformations = Transformations(
        [
            FilterRowsTransform(
                type=TransformType.FILTER_ROWS,
                operation="keep_rows",
                where=FilterGroup(
                    type="group",
                    operator="and",
                    children=[
                        FilterCondition(
                            type="condition",
                            column_id="strings",
                            operator="contains",
                            value=value,
                        )
                    ],
                ),
            )
        ]
    )

    polars_code = python_print_transforms(
        "my_df",
        my_df.columns,
        transformations.transforms,
        python_print_polars,
    )

    loc = {"pl": pl, "my_df": my_df.clone()}
    exec(polars_code, globals(), loc)
    code_result = loc["my_df_next"]

    nw_df = nw.from_native(my_df.clone(), eager_only=True).lazy()
    real_result = (
        _apply_transforms(
            nw_df,
            NarwhalsTransformHandler(),
            transformations,
        )
        .collect()
        .to_native()
    )

    pl_testing.assert_frame_equal(code_result, real_result)


@given(
    transform=create_transform_strategy(
        defined_column_id,
        string_column_id=st.just("strings"),
        bool_column_id=st.just("booleans"),
        comparison_column_id=st.sampled_from(
            ["integers", "floats", "dates", "times", "datetimes"]
        ),
        list_column_id=st.just("strings"),
        df_size=3,
    )
)
@settings(deadline=None)
@pytest.mark.skipif(
    not DependencyManager.ibis.has(), reason="ibis not installed"
)
@pytest.mark.xfail(reason="Ibis printing code is not well supported")
def test_print_code_result_matches_actual_transform_ibis(
    transform: Transform,
):
    import ibis

    my_df = ibis.memtable(
        {
            "strings": ["a", "b", "c"],
            "integers": [1, 2, 3],
            "floats": [1.0, 2.0, 3.0],
            "booleans": [True, False, True],
            "dates": [datetime.date(2021, 1, 1)] * 3,
            "times": [datetime.time(0, 0, 0)] * 3,
            "datetimes": [datetime.datetime(2021, 1, 1)] * 3,
            "mixed": [1, 2, 3],
            "dicts": [{"a": 1}, {"b": 2}, {"c": 3}],
            "lists": [[1, 2], [3, 4], [5, 6]],
        }
    )

    # Exclude shuffle and sample rows
    assume(
        transform.type
        not in {TransformType.SHUFFLE_ROWS, TransformType.SAMPLE_ROWS}
    )
    # Exclude boolean columns in filter rows
    if transform.type == TransformType.FILTER_ROWS:
        assume(
            not any(
                condition.column_id == "booleans"
                for condition in transform.where.children
            )
        )
    # Skip column conversion with errors='ignore' - ibis coalesce has type precedence issues
    if transform.type == TransformType.COLUMN_CONVERSION:
        assume(transform.errors != "ignore")

    if transform.type == TransformType.PIVOT:
        assume(
            (
                set(transform.column_ids) & set(transform.index_column_ids)
                == set()
            )
            and (
                set(transform.column_ids) & set(transform.value_column_ids)
                == set()
            )
            and (
                set(transform.index_column_ids)
                & set(transform.value_column_ids)
                == set()
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.index_column_ids
            )
        )
        assume(
            not any(
                column_id in {"lists", "dicts", "datetimes"}
                for column_id in transform.value_column_ids
            )
        )

    try:
        nw_df = nw.from_native(my_df).lazy()
        result_nw = _apply_transforms(
            nw_df,
            NarwhalsTransformHandler(),
            Transformations([transform]),
        )
        # Keep as narwhals lazy frame to check if it's an Ibis backend
        real_result = result_nw.collect().to_native()
    except Exception:
        real_result = None

    # Only compare if this passes
    assume(real_result is not None)

    assert real_result is not None

    ibis_code = python_print_transforms(
        "my_df",
        list(my_df.columns),
        [transform],
        python_print_ibis,
    )
    assert ibis_code

    loc = {"ibis": ibis, "my_df": my_df}
    exec(ibis_code, {}, loc)
    code_result = loc.get("my_df_next")

    # For pivot transform the column order can be different, enforce column order by sorting.
    if transform.type == TransformType.PIVOT:
        code_result = code_result.select(*sorted(code_result.columns))
        real_result = real_result.select(*sorted(real_result.columns))

    print("code_result", code_result)
    print("real_result", real_result)

    assert real_result is not None
    assert code_result is not None


@pytest.mark.skipif(
    not DependencyManager.pandas.has() or not DependencyManager.polars.has(),
    reason="pandas or polars not installed",
)
class TestCombinedTransforms:
    @pytest.fixture
    def sample_data(self):
        return {"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]}

    @pytest.fixture
    def pd_dataframe(self, sample_data):
        import pandas as pd

        return pd.DataFrame(sample_data)

    @pytest.fixture
    def pl_dataframe(self, sample_data):
        import polars as pl

        return pl.DataFrame(sample_data)

    def _test_transforms(
        self, df: pl.DataFrame | pd.DataFrame, transforms: Transformations
    ):
        import pandas as pd
        import pandas.testing as pd_testing
        import polars as pl
        import polars.testing as pl_testing

        if isinstance(df, pl.DataFrame):
            print_func = python_print_polars
            testing_func = pl_testing.assert_frame_equal
            df_copy = df.clone()
        elif isinstance(df, pd.DataFrame):
            print_func = python_print_pandas
            testing_func = pd_testing.assert_frame_equal
            df_copy = df.copy()

        # Convert to narwhals and apply transforms
        nw_df = nw.from_native(df_copy, eager_only=True).lazy()
        result_nw = _apply_transforms(
            nw_df, NarwhalsTransformHandler(), transforms
        )
        result = result_nw.collect().to_native()

        code = python_print_transforms(
            "df", df.columns, transforms.transforms, print_func
        )

        # Apply code
        if isinstance(df, pl.DataFrame):
            loc = {"pl": pl, "df": df.clone()}
            exec(code, globals(), loc)
        elif isinstance(df, pd.DataFrame):
            loc = {"pd": pd, "df": df.copy()}
            exec(code, {}, loc)

        assert loc.get("df_next") is not None

        # Get the results
        code_result = loc.get("df_next")

        # For group_by transforms, the row order might differ even with maintain_order=True
        # Sort both dataframes by all columns before comparing
        has_groupby = any(
            t.type == TransformType.GROUP_BY for t in transforms.transforms
        )
        if has_groupby:
            if isinstance(code_result, pl.DataFrame):
                import polars.datatypes as pl_dtypes

                # Filter out columns with unhashable types (dicts, lists)
                sortable_cols = [
                    col
                    for col in code_result.columns
                    if code_result[col].dtype
                    not in (pl_dtypes.List, pl_dtypes.Struct, pl_dtypes.Object)
                ]
                if sortable_cols:
                    code_result = code_result.sort(sortable_cols)
                    result = result.sort(sortable_cols)
            elif isinstance(code_result, pd.DataFrame):
                # Filter out columns with unhashable types (dicts, lists)
                sortable_cols = [
                    col
                    for col in code_result.columns
                    if not any(
                        isinstance(val, (dict, list))
                        for val in code_result[col]
                        if val is not None
                    )
                ]
                if sortable_cols:
                    code_result = code_result.sort_values(
                        by=sortable_cols
                    ).reset_index(drop=True)
                    result = result.sort_values(by=sortable_cols).reset_index(
                        drop=True
                    )

        # Test that the result matches the actual result
        testing_func(code_result, result)

    def test_select_then_group_by(
        self, pl_dataframe: pl.DataFrame, pd_dataframe: pd.DataFrame
    ):
        # Apply a select, then a group by
        transforms = Transformations(
            [
                SelectColumnsTransform(
                    type=TransformType.SELECT_COLUMNS,
                    column_ids=["a", "b"],
                ),
                GroupByTransform(
                    type=TransformType.GROUP_BY,
                    column_ids=["a"],
                    drop_na=False,
                    aggregation="sum",
                    aggregation_column_ids=["b"],
                ),
            ]
        )

        self._test_transforms(pl_dataframe, transforms)
        self._test_transforms(pd_dataframe, transforms)

    def test_select_rename_groupby(
        self, pl_dataframe: pl.DataFrame, pd_dataframe: pd.DataFrame
    ):
        transforms = Transformations(
            [
                SelectColumnsTransform(
                    type=TransformType.SELECT_COLUMNS,
                    column_ids=["a", "b"],
                ),
                RenameColumnTransform(
                    type=TransformType.RENAME_COLUMN,
                    column_id="a",
                    new_column_id="x",
                ),
                GroupByTransform(
                    type=TransformType.GROUP_BY,
                    column_ids=["x"],
                    drop_na=False,
                    aggregation="mean",
                    aggregation_column_ids=["b"],
                ),
            ]
        )
        self._test_transforms(pl_dataframe, transforms)
        self._test_transforms(pd_dataframe, transforms)

    def test_select_then_aggregate(
        self, pl_dataframe: pl.DataFrame, pd_dataframe: pd.DataFrame
    ):
        """Test select columns followed by aggregate transform."""
        transforms = Transformations(
            [
                SelectColumnsTransform(
                    type=TransformType.SELECT_COLUMNS,
                    column_ids=["a", "b", "c"],
                ),
                AggregateTransform(
                    type=TransformType.AGGREGATE,
                    column_ids=["a", "b"],
                    aggregations=["sum"],
                ),
            ]
        )

        self._test_transforms(pl_dataframe, transforms)
        self._test_transforms(pd_dataframe, transforms)

    def test_pivot_count_fills_null_with_zero(self):
        """Test that pivot with count aggregation fills null values with 0.

        When pivoting, some combinations may not exist in the data, resulting
        in null values. For count and sum aggregations, these should be filled
        with 0 to match narwhals behavior.
        """
        import pandas as pd
        import polars as pl

        # Create data where pivot will produce nulls
        # Only 'x' has values for group 'a', only 'y' has values for group 'b'
        pd_df = pd.DataFrame(
            {
                "group": ["a", "a", "b", "b"],
                "category": ["x", "x", "y", "y"],
                "value": [1, 2, 3, 4],
            }
        )
        pl_df = pl.DataFrame(
            {
                "group": ["a", "a", "b", "b"],
                "category": ["x", "x", "y", "y"],
                "value": [1, 2, 3, 4],
            }
        )

        transforms = Transformations(
            [
                PivotTransform(
                    type=TransformType.PIVOT,
                    column_ids=["category"],
                    index_column_ids=["group"],
                    aggregation="count",
                    value_column_ids=["value"],
                ),
            ]
        )

        self._test_transforms(pd_df, transforms)
        self._test_transforms(pl_df, transforms)

    def test_pivot_sum_fills_null_with_zero(self):
        """Test that pivot with sum aggregation fills null values with 0."""
        import pandas as pd
        import polars as pl

        pd_df = pd.DataFrame(
            {
                "group": ["a", "a", "b", "b"],
                "category": ["x", "x", "y", "y"],
                "value": [1, 2, 3, 4],
            }
        )
        pl_df = pl.DataFrame(
            {
                "group": ["a", "a", "b", "b"],
                "category": ["x", "x", "y", "y"],
                "value": [1, 2, 3, 4],
            }
        )

        transforms = Transformations(
            [
                PivotTransform(
                    type=TransformType.PIVOT,
                    column_ids=["category"],
                    index_column_ids=["group"],
                    aggregation="sum",
                    value_column_ids=["value"],
                ),
            ]
        )

        self._test_transforms(pd_df, transforms)
        self._test_transforms(pl_df, transforms)