A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
from __future__ import annotations
from datetime import datetime, time, timedelta
from typing import TYPE_CHECKING, Any
from unittest.mock import patch
import pytest
from marimo._data.preview_column import (
_sanitize_data,
get_column_preview_dataset,
get_column_preview_for_dataframe,
get_column_preview_for_duckdb,
get_table_manager,
)
from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.charts.altair_transformer import (
register_transformers,
)
from marimo._runtime.commands import PreviewDatasetColumnCommand
from marimo._utils.platform import is_windows
from tests._data.mocks import create_dataframes
from tests._plugins.ui._impl.tables import geometry_fixtures as geo
from tests.mocks import snapshotter
from tests.utils import assert_serialize_roundtrip
if TYPE_CHECKING:
from collections.abc import Generator
HAS_DF_DEPS = DependencyManager.pandas.has() and DependencyManager.altair.has()
HAS_SQL_DEPS = (
DependencyManager.duckdb.has() and DependencyManager.altair.has()
)
snapshot = snapshotter(__file__)
@pytest.fixture
def mock_light_theme():
"""Mock the theme to always return 'light' for consistent test results."""
with patch("marimo._data.charts.get_current_theme", return_value="light"):
yield
# Run cleanup after all tests are done
@pytest.fixture(scope="module", autouse=True)
def cleanup() -> Generator[None, None, None]:
import duckdb
yield
duckdb.execute("""
DROP TABLE IF EXISTS tbl;
DROP TABLE IF EXISTS date_tbl;
DROP TABLE IF EXISTS datetime_tbl;
DROP TABLE IF EXISTS time_tbl;
DROP TABLE IF EXISTS bool_tbl;
DROP TABLE IF EXISTS large_tbl;
""")
@pytest.mark.skipif(
not HAS_DF_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
@pytest.mark.parametrize(
"df",
create_dataframes(
{"A": [1, 2, 3], "B": ["a", "a", "a"]},
),
)
def test_get_column_preview_for_dataframe(df: Any) -> None:
register_transformers()
for column_name in ["A", "B"]:
result = get_column_preview_for_dataframe(
df,
request=PreviewDatasetColumnCommand(
source="source",
table_name="table",
column_name=column_name,
source_type="local",
),
)
assert result is not None
assert result.table_name == "table"
assert result.column_name == column_name
assert result.chart_code is not None
assert result.chart_spec is not None
assert result.stats is not None
assert result.error is None
assert_serialize_roundtrip(result)
@pytest.mark.skipif(
not HAS_DF_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
@pytest.mark.usefixtures("mock_light_theme")
@pytest.mark.parametrize(
("column_name", "snapshot_prefix"),
[
("A", "column_preview_int"),
("B", "column_preview_str"),
("date_col", "column_preview_date"),
("float_col", "column_preview_float"),
("bool_col", "column_preview_bool"),
("category_col", "column_preview_categorical"),
],
)
def test_get_column_preview(column_name: str, snapshot_prefix: str) -> None:
import pandas as pd
register_transformers()
df = pd.DataFrame(
{
"A": [1, 2, 3],
"B": ["a", "a", "a"],
"date_col": [
pd.Timestamp("2021-01-01"),
pd.Timestamp("2021-01-02"),
pd.Timestamp("2021-01-03"),
],
"float_col": [1.1, 2.2, 3.3],
"bool_col": [True, False, True],
"category_col": pd.Categorical(["cat1", "cat2", "cat1"]),
}
)
# Patch DependencyManager to simulate vegafusion not being available
with patch("marimo._data.preview_column.DependencyManager") as mock_dm:
# Keep altair available but make vegafusion unavailable
mock_dm.altair.has.return_value = True
mock_dm.vegafusion.has.return_value = False
mock_dm.vl_convert_python.has.return_value = True
result = get_column_preview_dataset(
table=get_table_manager(df),
table_name="table",
column_name=column_name,
)
assert result is not None
assert result.chart_code is not None
assert result.chart_spec is not None
assert result.stats is not None
assert result.error is None
snapshot(f"{snapshot_prefix}_chart_code.txt", result.chart_code)
# Skip date_col because of timezone
if column_name != "date_col":
snapshot(f"{snapshot_prefix}_chart_spec.txt", result.chart_spec)
# Verify vegafusion was checked
mock_dm.vegafusion.has.assert_called_once()
assert_serialize_roundtrip(result)
result_with_vegafusion = get_column_preview_dataset(
table=get_table_manager(df),
table_name="table",
column_name=column_name,
)
assert result_with_vegafusion is not None
assert result_with_vegafusion.chart_code is not None
assert result_with_vegafusion.chart_spec is not None
assert result_with_vegafusion.stats is not None
assert result_with_vegafusion.error is None
# Skip date_col because of timezone
if column_name != "date_col":
snapshot(
f"{snapshot_prefix}_chart_spec_with_vegafusion.txt",
result_with_vegafusion.chart_spec,
)
assert result_with_vegafusion.chart_code == result.chart_code
assert result_with_vegafusion.chart_spec != result.chart_spec
assert_serialize_roundtrip(result_with_vegafusion)
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb() -> None:
import duckdb
# Create a table with a deterministic pattern
duckdb.execute("""
CREATE OR REPLACE TABLE tbl AS
SELECT
range AS id,
CAST(range % 2 AS INTEGER) AS outcome
FROM range(100)
""")
# Test preview for the 'outcome' column (alternating 0 and 1)
result = get_column_preview_for_duckdb(
fully_qualified_table_name="tbl",
column_name="outcome",
)
assert result is not None
assert result.stats is not None
assert result.error is None
assert_serialize_roundtrip(result)
# Check if summary contains expected statistics for the alternating pattern
assert result.stats.total == 100
assert result.stats.unique == 2
assert result.stats.mean == 0.5 # Exactly 0.5 due to alternating pattern
assert result.chart_spec is not None
# Test preview for the 'id' column (for comparison)
result_id = get_column_preview_for_duckdb(
fully_qualified_table_name="tbl",
column_name="id",
)
assert result_id is not None
assert result_id.stats is not None
assert result_id.error is None
assert result_id.chart_spec is not None
assert_serialize_roundtrip(result_id)
# Not implemented yet
assert result.chart_code is None
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
@pytest.mark.usefixtures("mock_light_theme")
def test_get_column_preview_for_duckdb_categorical() -> None:
import duckdb
# Test preview for a categorical column
duckdb.execute("""
CREATE OR REPLACE TABLE tbl AS
SELECT
CASE
WHEN range % 4 = 0 THEN 'A'
WHEN range % 4 = 1 THEN 'B'
WHEN range % 4 = 2 THEN 'C'
ELSE 'D'
END AS category
FROM range(100)
""")
result_categorical = get_column_preview_for_duckdb(
fully_qualified_table_name="tbl",
column_name="category",
)
assert result_categorical is not None
assert result_categorical.stats is not None
assert result_categorical.error is None
assert_serialize_roundtrip(result_categorical)
# Check if summary contains expected statistics for the categorical pattern
assert result_categorical.stats.total == 100
assert result_categorical.stats.unique == 4
assert result_categorical.stats.nulls == 0
assert result_categorical.chart_spec is not None
snapshot(
"column_preview_duckdb_categorical_chart_spec.txt",
result_categorical.chart_spec,
)
# Not implemented yet
assert result_categorical.chart_code is None
# Test works when fully qualified
assert (
get_column_preview_for_duckdb(
fully_qualified_table_name="tbl",
column_name="category",
).stats
== get_column_preview_for_duckdb(
fully_qualified_table_name="memory.main.tbl",
column_name="category",
).stats
)
@pytest.mark.requires("duckdb", "altair", "pyarrow", "vegafusion")
@pytest.mark.parametrize("column_type", ["DATE", "TIMESTAMP"])
@pytest.mark.parametrize("row_count", [0, 3])
def test_get_column_preview_for_duckdb_missing_dates(
column_type: str, row_count: int
) -> None:
import duckdb
table_name = "missing_dates_preview"
duckdb.execute(
f"CREATE OR REPLACE TABLE {table_name} AS "
f"SELECT NULL::{column_type} AS value FROM range({row_count})"
).fetchall()
try:
result = get_column_preview_for_duckdb(
fully_qualified_table_name=table_name,
column_name="value",
)
assert result is not None
assert result.error is None
assert result.chart_spec is not None
assert result.stats is not None
assert (result.stats.total, result.stats.nulls) == (
row_count,
row_count if row_count else None,
)
assert duckdb.sql("SELECT 42").fetchall() == [(42,)]
assert_serialize_roundtrip(result)
finally:
duckdb.execute(f"DROP TABLE {table_name}").fetchall()
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb_date() -> None:
import duckdb
# Test preview for a date column
duckdb.execute("""
CREATE OR REPLACE TABLE date_tbl AS
SELECT DATE '2023-01-01' + INTERVAL (range % 365) DAY AS date_col
FROM range(100)
""")
result_date = get_column_preview_for_duckdb(
fully_qualified_table_name="date_tbl",
column_name="date_col",
)
assert result_date is not None
assert result_date.stats is not None
assert result_date.error is None
assert_serialize_roundtrip(result_date)
# Check if summary contains expected statistics for the date pattern
assert result_date.stats.total == 100
assert result_date.stats.unique == 100
assert result_date.stats.nulls == 0
assert result_date.stats.min == datetime(2023, 1, 1, 0, 0)
assert result_date.stats.max == datetime(2023, 4, 10, 0, 0)
assert result_date.chart_spec is not None
# No chart_spec snapshot because of date timezone
# Not implemented yet
assert result_date.chart_code is None
# Test works when fully qualified
assert (
get_column_preview_for_duckdb(
fully_qualified_table_name="date_tbl",
column_name="date_col",
).stats
== get_column_preview_for_duckdb(
fully_qualified_table_name="memory.main.date_tbl",
column_name="date_col",
).stats
)
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb_datetime() -> None:
import duckdb
# Test preview for a datetime column
duckdb.execute("""
CREATE OR REPLACE TABLE datetime_tbl AS
SELECT TIMESTAMP '2023-01-01 00:00:00' +
INTERVAL (range % 365) DAY +
INTERVAL (range % 24) HOUR +
INTERVAL (range % 60) MINUTE AS datetime_col
FROM range(100)
""")
result_datetime = get_column_preview_for_duckdb(
fully_qualified_table_name="datetime_tbl",
column_name="datetime_col",
)
assert result_datetime is not None
assert result_datetime.stats is not None
assert result_datetime.error is None
assert_serialize_roundtrip(result_datetime)
# Check if summary contains expected statistics for the datetime pattern
assert result_datetime.stats.total == 100
assert result_datetime.stats.unique == 100
assert result_datetime.stats.nulls == 0
assert result_datetime.stats.min == datetime(2023, 1, 1, 0, 0)
assert result_datetime.stats.max == datetime(2023, 4, 10, 3, 39)
assert result_datetime.chart_spec is not None
# Not implemented yet
assert result_datetime.chart_code is None
# No chart_spec snapshot because of date timezone
# Test works when fully qualified
assert (
get_column_preview_for_duckdb(
fully_qualified_table_name="datetime_tbl",
column_name="datetime_col",
).stats
== get_column_preview_for_duckdb(
fully_qualified_table_name="memory.main.datetime_tbl",
column_name="datetime_col",
).stats
)
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb_time() -> None:
import duckdb
# Test preview for a time column
duckdb.execute("""
CREATE OR REPLACE TABLE time_tbl AS
SELECT TIME '00:00:00' +
INTERVAL (range % 24) HOUR +
INTERVAL (range % 60) MINUTE AS time_col
FROM range(100)
""")
result_time = get_column_preview_for_duckdb(
fully_qualified_table_name="time_tbl",
column_name="time_col",
)
assert result_time is not None
assert result_time.stats is not None
assert result_time.error is None
assert_serialize_roundtrip(result_time)
# Check if summary contains expected statistics for the time pattern
assert result_time.stats.total == 100
assert result_time.stats.unique == 100
assert result_time.stats.nulls == 0
assert result_time.stats.min == time(0, 0)
assert result_time.stats.max == time(23, 47)
# Time is not handled yet
assert result_time.chart_spec is None
# Not implemented yet
assert result_time.chart_code is None
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb_bool() -> None:
import duckdb
# Test preview for a boolean column
duckdb.execute("""
CREATE OR REPLACE TABLE bool_tbl AS
SELECT range % 2 = 0 AS bool_col
FROM range(100)
""")
result_bool = get_column_preview_for_duckdb(
fully_qualified_table_name="bool_tbl",
column_name="bool_col",
)
assert result_bool is not None
assert result_bool.stats is not None
assert result_bool.error is None
assert_serialize_roundtrip(result_bool)
# Check if summary contains expected statistics for the boolean pattern
assert result_bool.stats.total == 100
assert result_bool.stats.unique == 2
assert result_bool.stats.nulls == 0
assert result_bool.stats.true == 50
assert result_bool.stats.false == 50
assert result_bool.chart_spec is not None
snapshot(
"column_preview_duckdb_bool_chart_spec.txt", result_bool.chart_spec
)
# Not implemented yet
assert result_bool.chart_code is None
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
@pytest.mark.skipif(is_windows(), reason="Windows encodes base64 differently")
def test_get_column_preview_for_duckdb_over_limit() -> None:
import duckdb
from marimo._data.preview_column import CHART_MAX_ROWS
# Create a table with more rows than the chart limit
duckdb.execute(f"""
CREATE OR REPLACE TABLE large_tbl AS
SELECT
range AS id,
range % 5 AS category
FROM range({CHART_MAX_ROWS + 100})
""")
# Test preview for a column in a table that exceeds the row limit
result = get_column_preview_for_duckdb(
fully_qualified_table_name="large_tbl",
column_name="category",
)
assert result is not None
assert result.stats is not None
assert (
result.error == "Too many rows, vegafusion required to render charts"
)
assert result.missing_packages == ["vegafusion", "vl_convert_python"]
assert result.chart_spec is None
assert_serialize_roundtrip(result)
# Not implemented yet
assert result.chart_code is None
@pytest.mark.skipif(
not HAS_SQL_DEPS, reason="optional dependencies not installed"
)
def test_get_column_preview_for_duckdb_stats_total_none() -> None:
"""Test that when stats.total is None, no chart is generated and no crash occurs."""
from marimo._data.models import ColumnStats
with (
patch(
"marimo._data.preview_column.get_sql_stats",
return_value=ColumnStats(),
),
patch(
"marimo._data.preview_column.get_column_type",
return_value="integer",
),
):
result = get_column_preview_for_duckdb(
fully_qualified_table_name="tbl",
column_name="id",
)
assert result is not None
assert result.stats is not None
assert result.stats.total is None
# No chart should be generated when total is None
assert result.chart_spec is None
assert result.error is None
@pytest.mark.skipif(
not DependencyManager.narwhals.has() or not DependencyManager.polars.has(),
reason="narwhals and polars not installed",
)
def test_sanitize_dtypes() -> None:
import narwhals as nw
import polars as pl
df = pl.DataFrame(
{"int128_col": [1, 2, 3], "cat_col": ["A", "B", "A"]},
schema={"int128_col": pl.Int128, "cat_col": pl.Categorical},
)
nw_df = nw.from_native(df)
# Sanitize the dtypes
result = _sanitize_data(nw_df, "cat_col")
assert result.collect_schema()["cat_col"] == nw.String
result = _sanitize_data(nw_df, "int128_col")
assert result.collect_schema()["int128_col"] == nw.Int64
@pytest.mark.skipif(
not DependencyManager.narwhals.has() or not DependencyManager.polars.has(),
reason="narwhals and polars not installed",
)
def test_sanitize_dtypes_enum() -> None:
import narwhals as nw
import polars as pl
df = pl.DataFrame(
{
"enum_col": ["A", "B", "A"],
},
schema={"enum_col": pl.Enum(["A", "B"])},
)
nw_df = nw.from_native(df)
result = _sanitize_data(nw_df, "enum_col")
assert result.collect_schema()["enum_col"] == nw.String
lazy_df = nw_df.lazy()
result = _sanitize_data(lazy_df, "enum_col")
assert result.collect_schema()["enum_col"] == nw.String
@pytest.mark.skipif(
not DependencyManager.narwhals.has()
or not DependencyManager.polars.has()
or not DependencyManager.altair.has(),
reason="narwhals, polars and altair not installed",
)
def test_preview_column_struct_skips_chart() -> None:
import polars as pl
df = pl.DataFrame({"struct_col": [{"a": 1, "b": "x"}, {"a": 2, "b": "y"}]})
with patch("marimo._data.preview_column.LOGGER") as mock_logger:
result = get_column_preview_dataset(
table=get_table_manager(df),
table_name="table",
column_name="struct_col",
)
assert result is not None
assert result.error is None
assert result.chart_spec is None
assert result.chart_code is None
assert result.stats is not None
mock_logger.warning.assert_not_called()
@pytest.mark.requires("altair")
@pytest.mark.requires("geopandas")
def test_preview_column_geometry_skips_chart() -> None:
manager = get_table_manager(geo.gdf_point_known_crs())
assert manager is not None
result = get_column_preview_dataset(
table=manager,
table_name="table",
column_name="geometry",
)
assert result.error is None
assert result.chart_spec is None
assert result.chart_code is None
assert result.stats is not None
@pytest.mark.parametrize(
"df",
create_dataframes(
{
"duration_weeks": [timedelta(weeks=1), timedelta(weeks=2)],
"duration_days": [timedelta(days=1), timedelta(days=2)],
"duration_hours": [timedelta(hours=1), timedelta(hours=2)],
"duration_minutes": [timedelta(minutes=1), timedelta(minutes=2)],
"duration_seconds": [timedelta(seconds=1), timedelta(seconds=2)],
"duration_milliseconds": [
timedelta(milliseconds=1),
timedelta(milliseconds=2),
],
"duration_microseconds": [
timedelta(microseconds=1),
timedelta(microseconds=2),
],
},
exclude=["pyarrow", "duckdb", "ibis"],
),
)
def test_preview_column_duration_dtype(df) -> None:
columns = (
df.collect_schema().names()
if hasattr(df, "collect_schema")
else df.columns
)
for column_name in columns:
result = get_column_preview_dataset(
table=get_table_manager(df),
table_name="table",
column_name=column_name,
)
assert result is not None
assert result.chart_code is not None
assert result.chart_spec is not None
assert_serialize_roundtrip(result)