A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
from __future__ import annotations
import datetime
import json
import unittest
from typing import Any
import pytest
from marimo._data.models import BinValue, ColumnStats
from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.table import SortArgs
from marimo._plugins.ui._impl.tables.format import FormatMapping
from marimo._plugins.ui._impl.tables.ibis_table import (
IbisTableManagerFactory,
)
from marimo._plugins.ui._impl.tables.table_manager import TableManager
from tests.mocks import snapshotter
HAS_DEPS = DependencyManager.ibis.has()
snapshot = snapshotter(__file__)
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
class TestIbisTableManagerFactory(unittest.TestCase):
def get_complex_data(self) -> TableManager[Any]:
import ibis
complex_data = ibis.memtable(
{
"strings": ["a", "b", "c"],
"bool": [True, False, True],
"int": [1, 2, 3],
"float": [1.0, 2.0, 3.0],
"date": [
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 2),
datetime.date(2021, 1, 3),
],
"datetime": [
datetime.datetime(2021, 1, 1),
datetime.datetime(2021, 1, 2),
datetime.datetime(2021, 1, 3),
],
"time": [
datetime.time(1, 2, 3),
datetime.time(4, 5, 6),
datetime.time(7, 8, 9),
],
"nulls": [None, "data", None],
}
)
return self.factory.create()(complex_data)
def setUp(self) -> None:
import ibis
self.factory = IbisTableManagerFactory()
self.data = ibis.memtable(
{
"A": [1, 2, 3],
"B": ["a", "b", "c"],
"C": [1.0, 2.0, 3.0],
"D": [True, False, True],
"E": [
datetime.datetime(2021, 1, 1),
datetime.datetime(2021, 1, 2),
datetime.datetime(2021, 1, 3),
],
}
)
self.manager = self.factory.create()(self.data)
def test_package_name(self) -> None:
assert self.factory.package_name() == "ibis"
def test_to_csv(self) -> None:
assert isinstance(self.manager.to_csv(), bytes)
complex_data = self.get_complex_data()
data = complex_data.to_csv_str()
assert isinstance(data, str)
snapshot("ibis.csv", data)
def test_to_json(self) -> None:
assert isinstance(self.manager.to_json(), bytes)
complex_data = self.get_complex_data()
data = complex_data.to_json_str()
assert isinstance(data, str)
snapshot("ibis.json", data)
def test_to_json_format_mapping(self) -> None:
import ibis
table = ibis.memtable({"int": [1, 2, 3]}, schema={"int": "int64"})
data = self.factory.create()(table)
format_mapping: FormatMapping = {"int": lambda x: x * 2}
json_data = data.to_json_str(format_mapping)
json_object = json.loads(json_data)
assert json_object == [{"int": 2}, {"int": 4}, {"int": 6}]
def test_complex_data_field_types(self) -> None:
complex_data = self.get_complex_data()
field_types = complex_data.get_field_types()
snapshot("ibis.field_types.json", json.dumps(field_types))
def test_select_rows(self) -> None:
indices = [0, 2]
selected_manager = self.manager.select_rows(indices)
assert selected_manager.data.to_dict(as_series=False) == {
"A": [1, 3],
"B": ["a", "c"],
"C": [1.0, 3.0],
"D": [True, True],
"E": [
datetime.datetime(2021, 1, 1),
datetime.datetime(2021, 1, 3),
],
}
def test_select_columns(self) -> None:
columns = ["A"]
selected_manager = self.manager.select_columns(columns)
assert selected_manager.data.collect().to_dict(as_series=False) == {
"A": [1, 2, 3],
}
def test_drop_columns(self) -> None:
columns = ["A"]
dropped_manager = self.manager.drop_columns(columns)
assert dropped_manager.data.columns == ["B", "C", "D", "E"]
def test_get_row_headers(self) -> None:
expected_headers = []
assert self.manager.get_row_headers() == expected_headers
def test_is_type(self) -> None:
assert self.manager.is_type(self.data)
assert not self.manager.is_type("not a table")
def test_get_field_types(self) -> None:
expected_field_types = [
("A", ("integer", "int64")),
("B", ("string", "string")),
("C", ("number", "float64")),
("D", ("boolean", "boolean")),
("E", ("datetime", "timestamp")),
]
assert self.manager.get_field_types() == expected_field_types
def test_limit(self) -> None:
limited_manager = self.manager.take(1, 0)
assert limited_manager.get_num_rows() == 1
def test_take(self) -> None:
def as_list(df: Any) -> list[Any]:
return df.to_dict(as_series=False)["A"]
assert as_list(self.manager.take(1, 0).data.collect()) == [1]
assert as_list(self.manager.take(2, 0).data.collect()) == [
1,
2,
]
assert as_list(
self.manager.take(2, 1).select_columns(["A"]).data.collect()
) == [
2,
3,
]
assert as_list(
self.manager.take(2, 2).select_columns(["A"]).data.collect()
) == [
3,
]
def test_to_parquet(self) -> None:
assert isinstance(self.manager.to_parquet(), bytes)
def test_take_zero(self) -> None:
limited_manager = self.manager.take(0, 0)
assert limited_manager.data.collect().to_dict(as_series=False) == {
"A": [],
"B": [],
"C": [],
"D": [],
"E": [],
}
def test_take_negative(self) -> None:
with pytest.raises(ValueError):
self.manager.take(-1, 0)
def test_take_negative_offset(self) -> None:
with pytest.raises(ValueError):
self.manager.take(1, -1)
def test_take_out_of_bounds(self) -> None:
# Too large of page
assert self.manager.take(10, 0).get_num_rows() == 3
assert self.manager.get_num_rows() == 3
# Too large of page and offset
assert self.manager.take(10, 10).get_num_rows() == 0
def test_stats_integer(self) -> None:
column = "A"
stats = self.manager.get_stats(column)
assert stats == ColumnStats(
total=3,
nulls=0,
min=1,
max=3,
mean=2.0,
median=2.0,
unique=3,
std=1.0,
)
def test_stats_string(self) -> None:
column = "B"
stats = self.manager.get_stats(column)
assert stats == ColumnStats(
total=3,
nulls=0,
unique=3,
)
def test_sort_values(self) -> None:
sorted_manager = self.manager.sort_values(
[SortArgs(by="A", descending=True)]
)
assert sorted_manager.data.collect().to_dict(as_series=False) == {
"A": [3, 2, 1],
"B": ["c", "b", "a"],
"C": [3.0, 2.0, 1.0],
"D": [True, False, True],
"E": [
datetime.datetime(2021, 1, 3),
datetime.datetime(2021, 1, 2),
datetime.datetime(2021, 1, 1),
],
}
def test_get_unique_column_values(self) -> None:
column = "A"
unique_values = self.manager.get_unique_column_values(column)
assert sorted(unique_values) == [1, 2, 3]
def test_get_sample_values(self) -> None:
sample_values = self.manager.get_sample_values("A")
assert sample_values == []
def test_search(self) -> None:
import ibis
df = ibis.memtable(
{
"A": [1, 2, 3],
"B": ["foo", "bar", "baz"],
"C": [True, False, True],
}
)
manager = self.factory.create()(df)
assert manager.search("foo").get_num_rows() == 1
assert manager.search("a").get_num_rows() == 2
assert manager.search("true").get_num_rows() == 2
assert manager.search("food").get_num_rows() == 0
def test_apply_formatting(self) -> None:
format_mapping: FormatMapping = {
"A": lambda x: x * 2,
"B": lambda x: x.upper(),
"C": lambda x: f"{x:.2f}",
"D": lambda x: not x,
"E": lambda x: x.strftime("%Y-%m-%d"),
}
formatted_data = self.manager.apply_formatting(format_mapping)
assert formatted_data.data.to_dict(as_series=False) == {
"A": [2, 4, 6],
"B": ["A", "B", "C"],
"C": ["1.00", "2.00", "3.00"],
"D": [False, True, False],
"E": ["2021-01-01", "2021-01-02", "2021-01-03"],
}
def test_empty_table(self) -> None:
import ibis
empty_table = ibis.memtable({"A": []})
empty_manager = self.factory.create()(empty_table)
assert empty_manager.get_num_rows() == 0
def test_table_with_all_null_column(self) -> None:
import ibis
table = ibis.memtable({"A": [1, 2, 3], "B": [None, None, None]})
manager = self.factory.create()(table)
stats = manager.get_stats("B")
assert stats.nulls == 3
assert stats.total == 3
def test_table_with_mixed_types(self) -> None:
import ibis
table = ibis.memtable({"A": [1, "two", 3.0, True]})
manager = self.factory.create()(table)
assert manager.get_field_type("A") == ("unknown", "unknown")
def test_search_treats_query_as_literal(self) -> None:
import ibis
table = ibis.memtable(
{
"A": [
"apple",
"banana",
"C++",
"c++ guide",
"ΟΣ",
"ος",
"οσ",
"İstanbul",
]
}
)
manager = self.factory.create()(table)
assert manager.search("^[ab]").get_num_rows() == 0
assert manager.search("c++").get_num_rows() == 2
assert manager.search("C++ G").get_num_rows() == 1
assert manager.search("a.b").get_num_rows() == 0
assert manager.search("Σ").get_num_rows() == 3
assert manager.search("İ").get_num_rows() == 1
def test_sort_values_with_nulls(self) -> None:
import ibis
table = ibis.memtable({"A": [3, 1, None, 2]})
manager = self.factory.create()(table)
# Descending true
sorted_manager = manager.sort_values(
[SortArgs(by="A", descending=True)]
)
sorted_data = sorted_manager.data.collect().to_dict(as_series=False)[
"A"
]
assert sorted_data == [
3.0,
2.0,
1.0,
None,
]
# Descending false
sorted_manager = manager.sort_values(
[SortArgs(by="A", descending=False)]
)
sorted_data = sorted_manager.data.collect().to_dict(as_series=False)[
"A"
]
assert sorted_data == [
1.0,
2.0,
3.0,
None,
]
def test_calculate_top_k_rows(self) -> None:
import ibis
table = ibis.memtable({"A": [2, 3, 3], "B": ["a", "b", "c"]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert result == [(3, 2), (2, 1)]
# Test equal counts with k limit
table = ibis.memtable({"A": [1, 1, 2, 2, 2, 3]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 2)
assert result == [(2, 3), (1, 2)]
def test_calculate_top_k_rows_nulls(self) -> None:
import ibis
# Test single null value
table = ibis.memtable({"A": [3, None, None]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert result == [
(None, 2),
(3, 1),
]
# Test all null values
table = ibis.memtable({"A": [None, None, None]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert result == [(None, 3)]
# Test mixed values with nulls
table = ibis.memtable({"A": [1, None, 2, 2, None, 3, None]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert len(result) == 4
assert result[0][0] is None
assert result[0][1] == 3
assert set(result[1:]) == {(1, 1), (2, 2), (3, 1)}
def test_calculate_top_k_rows_nested_lists(self) -> None:
import ibis
# Test nested lists
table = ibis.memtable({"A": [[1, 2], [1, 2], [3, 4]]})
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert result == [([1, 2], 2), ([3, 4], 1)]
def test_calculate_top_k_rows_dicts(self) -> None:
import ibis
# Test dicts
table = ibis.memtable(
{"A": [{"a": 1, "b": 2}, {"a": 1, "b": 2}, {"a": 3, "b": 4}]}
)
manager = self.factory.create()(table)
result = manager.calculate_top_k_rows("A", 10)
assert result == [({"a": 1, "b": 2}, 2), ({"a": 3, "b": 4}, 1)]
def test_get_bin_values(self) -> None:
import ibis
table = ibis.memtable(
{
"int": [3, 5, -1, 6, 8, 10, 11, 23, 25],
"float": [3.2, 4.8, -1.0, 8.0, 7.5, 9.5, 11.0, None, 24.8],
"string": ["a", "b", "c", "d", "e", "f", "g", "h", "i"],
}
)
manager = self.factory.create()(table)
result = manager.get_bin_values("int", 5)
assert result == [
BinValue(bin_start=-1.0, bin_end=4.2, count=2),
BinValue(bin_start=4.2, bin_end=9.4, count=3),
BinValue(bin_start=9.4, bin_end=pytest.approx(14.6), count=2),
BinValue(bin_start=pytest.approx(14.6), bin_end=19.8, count=0),
BinValue(bin_start=19.8, bin_end=25.0, count=2),
]
result = manager.get_bin_values("float", 3)
print(result)
assert result == [
BinValue(bin_start=-1.0, bin_end=7.6, count=4),
BinValue(bin_start=7.6, bin_end=16.2, count=3),
BinValue(bin_start=16.2, bin_end=24.8, count=1),
]
# Not supported for other column types
result = manager.get_bin_values("string", 3)
assert result == []
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
class TestTemporalColSummaries(unittest.TestCase):
"""Tests are quite flaky with exact values, so we test length and counts"""
manager: TableManager[Any]
def setUp(self) -> None:
import ibis
self.factory = IbisTableManagerFactory()
self.data = ibis.memtable(
{
"date": [
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 2),
datetime.date(2021, 1, 3),
datetime.date(2021, 1, 4),
datetime.date(2021, 1, 5),
],
"datetime": [
datetime.datetime(2021, 1, 1),
datetime.datetime(2021, 1, 2),
datetime.datetime(2021, 1, 3),
datetime.datetime(2021, 1, 4),
datetime.datetime(2021, 1, 5),
],
"datetime_with_tz": [
datetime.datetime(2021, 1, 1, tzinfo=datetime.UTC),
datetime.datetime(2021, 1, 2, tzinfo=datetime.UTC),
datetime.datetime(2021, 1, 3, tzinfo=datetime.UTC),
datetime.datetime(2021, 1, 4, tzinfo=datetime.UTC),
datetime.datetime(2021, 1, 5, tzinfo=datetime.UTC),
],
"time": [
datetime.time(1, 2, 3),
datetime.time(4, 5, 6),
datetime.time(7, 8, 9),
datetime.time(10, 11, 12),
datetime.time(13, 14, 15),
],
"dates_multiple": [
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 1),
datetime.date(2021, 1, 1),
],
"timedelta": [
datetime.timedelta(days=1),
datetime.timedelta(days=2),
datetime.timedelta(days=3),
datetime.timedelta(days=4),
datetime.timedelta(days=5),
],
}
)
self.manager = self.factory.create()(self.data)
def test_date_column(self) -> None:
bin_values = self.manager.get_bin_values("date", 3)
assert len(bin_values) == 3
assert bin_values[0].count == 2
assert bin_values[1].count == 1
assert bin_values[2].count == 2
def test_datetime_column(self) -> None:
bin_values = self.manager.get_bin_values("datetime", 3)
assert len(bin_values) == 3
assert bin_values[0].count == 2
assert bin_values[1].count == 1
assert bin_values[2].count == 2
def test_time_column(self) -> None:
bin_values = self.manager.get_bin_values("time", 3)
assert len(bin_values) == 3
assert bin_values[0].count == 2
assert bin_values[1].count == 1
assert bin_values[2].count == 2
def test_dates_multiple(self) -> None:
bin_values = self.manager.get_bin_values("dates_multiple", 3)
assert len(bin_values) == 1
assert bin_values[0].count == 5
@pytest.mark.xfail(reason="datetime with tz is not supported")
def test_datetime_with_tz(self) -> None:
bin_values = self.manager.get_bin_values("datetime_with_tz", 3)
assert bin_values == [
BinValue(
bin_start=datetime.datetime(
2021, 1, 1, 8, 0, 0, tzinfo=datetime.UTC
),
bin_end=datetime.datetime(
2021, 1, 2, 16, 0, 0, tzinfo=datetime.UTC
),
count=2,
),
BinValue(
bin_start=datetime.datetime(
2021, 1, 2, 16, 0, 0, tzinfo=datetime.UTC
),
bin_end=datetime.datetime(
2021, 1, 4, 0, 0, 0, tzinfo=datetime.UTC
),
count=1,
),
BinValue(
bin_start=datetime.datetime(
2021, 1, 4, 0, 0, 0, tzinfo=datetime.UTC
),
bin_end=datetime.datetime(
2021, 1, 5, 8, 0, 0, tzinfo=datetime.UTC
),
count=2,
),
]
@pytest.mark.xfail(reason="timedelta is not supported")
def test_timedelta_column(self) -> None:
bin_values = self.manager.get_bin_values("timedelta", 3)
assert bin_values == [
BinValue(
bin_start=datetime.timedelta(days=1),
bin_end=datetime.timedelta(days=2),
count=2,
)
]
result = self.manager.calculate_top_k_rows("A", 2)
assert result == [(None, 3), (2, 2)]