A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
from typing import TYPE_CHECKING
from unittest import mock
import pytest
from marimo._data.models import DataTable, DataTableColumn
from marimo._dependencies.dependencies import DependencyManager
from marimo._sql.engines.pyiceberg import PyIcebergEngine
from marimo._sql.engines.types import (
NO_SCHEMA_NAME,
EngineCatalog,
QueryEngine,
)
from marimo._types.ids import VariableName
HAS_PYICEBERG = DependencyManager.pyiceberg.has()
if TYPE_CHECKING:
from collections.abc import Generator
from pyiceberg.catalog import Catalog
@pytest.fixture
def memory_catalog() -> Generator[Catalog, None, None]:
"""Create a mock PyIceberg catalog for testing."""
if not HAS_PYICEBERG:
yield mock.MagicMock()
return
from pyiceberg.catalog.memory import InMemoryCatalog
from pyiceberg.schema import Schema as IcebergSchema
from pyiceberg.types import (
BooleanType,
IntegerType,
NestedField,
StringType,
)
# Create mock catalog
catalog = InMemoryCatalog("test_catalog")
# Create namespaces, including nested ones
catalog.create_namespace("default")
catalog.create_namespace("test_namespace")
catalog.create_namespace(("top", "nested"))
catalog.create_namespace(("top", "nested", "deep"))
# Create actual schema object with real fields
schema = IcebergSchema(
fields=[
NestedField(
field_id=1, name="id", field_type=IntegerType(), required=True
),
NestedField(
field_id=2, name="name", field_type=StringType(), required=True
),
NestedField(
field_id=3,
name="active",
field_type=BooleanType(),
required=False,
),
]
)
catalog.create_table(("default", "table1"), schema)
catalog.create_table(("default", "table2"), schema)
catalog.create_table(("test_namespace", "table3"), schema)
catalog.create_table(("top", "nested", "table4"), schema)
catalog.create_table(("top", "nested", "deep", "table5"), schema)
assert len(catalog.list_tables("default")) == 2
assert len(catalog.list_tables("test_namespace")) == 1
assert len(catalog.list_tables(("top", "nested"))) == 1
assert len(catalog.list_tables(("top", "nested", "deep"))) == 1
yield catalog
catalog.drop_table(("default", "table1"))
catalog.drop_table(("default", "table2"))
catalog.drop_table(("test_namespace", "table3"))
catalog.drop_table(("top", "nested", "table4"))
catalog.drop_table(("top", "nested", "deep", "table5"))
def get_expected_table(
table_name: str, include_table_details: bool = True
) -> DataTable:
"""Return expected table structure for tests."""
return DataTable(
source_type="catalog",
source="iceberg",
name=table_name,
num_rows=None,
num_columns=3 if include_table_details else None,
variable_name=None,
engine=VariableName("my_iceberg"),
type="table",
primary_keys=[] if not include_table_details else None,
indexes=[] if not include_table_details else None,
columns=[
DataTableColumn(
name="id",
type="integer",
external_type="int",
sample_values=[],
),
DataTableColumn(
name="name",
type="string",
external_type="string",
sample_values=[],
),
DataTableColumn(
name="active",
type="boolean",
external_type="boolean",
sample_values=[],
),
]
if include_table_details
else [],
)
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_engine_compatibility() -> None:
"""Test engine compatibility checks."""
from pyiceberg.catalog.memory import InMemoryCatalog
obj = object()
mock_catalog = InMemoryCatalog("test_catalog")
assert PyIcebergEngine.is_compatible(mock_catalog)
assert not PyIcebergEngine.is_compatible(obj)
engine = PyIcebergEngine(
mock_catalog, engine_name=VariableName("my_iceberg")
)
assert isinstance(engine, PyIcebergEngine)
assert isinstance(engine, EngineCatalog)
assert not isinstance(engine, QueryEngine)
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_engine_name_initialization(memory_catalog: Catalog) -> None:
"""Test engine name initialization."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
assert engine._engine_name == VariableName("my_iceberg")
# Test default name
engine = PyIcebergEngine(memory_catalog)
assert engine._engine_name is None
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_engine_source_and_dialect(memory_catalog: Catalog) -> None:
"""Test PyIcebergEngine source and dialect properties."""
engine = PyIcebergEngine(memory_catalog)
assert engine.source == "iceberg"
assert engine.dialect == "iceberg"
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_engine_get_database_name(memory_catalog: Catalog) -> None:
"""Test PyIcebergEngine get_database_name."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
assert engine.get_default_database() is None
assert engine.get_default_schema() is None
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_execute(memory_catalog: Catalog) -> None:
"""Test PyIceberg execute raises NotImplementedError."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
assert isinstance(engine, EngineCatalog)
assert not isinstance(engine, QueryEngine)
with pytest.raises(AttributeError):
engine.execute("SELECT * FROM table")
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_table_details(memory_catalog: Catalog) -> None:
"""Test PyIcebergEngine get_table_details method."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
table = engine.get_table_details(
table_name="table1",
schema_name=NO_SCHEMA_NAME,
database_name="default",
)
assert table is not None
assert table.source == "iceberg"
assert table.name == "table1"
assert table.num_columns == 3
assert len(table.columns) == 3
assert table.columns[0].name == "id"
assert table.columns[0].type == "integer"
assert table.columns[1].name == "name"
assert table.columns[1].type == "string"
assert table.columns[2].name == "active"
assert table.columns[2].type == "boolean"
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_tables_in_schema(memory_catalog: Catalog) -> None:
"""Test PyIcebergEngine get_tables_in_schema method."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
# Test with include_table_details=True
tables = engine.get_tables_in_schema(
schema="unused", database="default", include_table_details=True
)
assert isinstance(tables, list)
assert len(tables) == 2
assert tables[0].name == "table1"
assert tables[1].name == "table2"
assert tables[0].num_columns == 3
assert tables[1].num_columns == 3
# Test with include_table_details=False
tables = engine.get_tables_in_schema(
schema="unused", database="default", include_table_details=False
)
assert isinstance(tables, list)
assert len(tables) == 2
assert tables[0].name == "table1"
assert tables[1].name == "table2"
assert tables[0].num_columns is None
assert tables[1].num_columns is None
assert tables[0].columns == []
assert tables[1].columns == []
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_databases(memory_catalog: Catalog) -> None:
"""Each top-level namespace is a Database; sub-namespaces are recursive
child Schemas."""
engine_name = VariableName("my_iceberg")
engine = PyIcebergEngine(memory_catalog, engine_name=engine_name)
# Test with all parameters True (eager)
databases = engine.get_databases(
include_schemas=True, include_tables=True, include_table_details=True
)
assert isinstance(databases, list)
# Only top-level namespaces become Databases.
by_name = {db.name: db for db in databases}
assert set(by_name) == {"default", "test_namespace", "top"}
# "default" has its own tables in a schemaless Schema, no sub-namespaces.
default_schemas = {s.name: s for s in by_name["default"].schemas}
assert by_name["default"].dialect == "iceberg"
assert set(default_schemas) == {NO_SCHEMA_NAME}
assert len(default_schemas[NO_SCHEMA_NAME].tables) == 2
assert default_schemas[NO_SCHEMA_NAME].child_schemas == []
assert {s.name for s in by_name["test_namespace"].schemas} == {
NO_SCHEMA_NAME
}
# "top" has no tables of its own but contains the "nested" sub-namespace.
top_schemas = {s.name: s for s in by_name["top"].schemas}
assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
assert top_schemas[NO_SCHEMA_NAME].tables == []
nested = top_schemas["nested"]
assert [t.name for t in nested.tables] == ["table4"]
# "nested" recursively contains "deep", which holds "table5".
deep_by_name = {s.name: s for s in nested.child_schemas}
assert set(deep_by_name) == {"deep"}
assert [t.name for t in deep_by_name["deep"].tables] == ["table5"]
# Test with include_tables=False (schemas listed, tables deferred)
databases = engine.get_databases(
include_schemas=True, include_tables=False, include_table_details=True
)
by_name = {db.name: db for db in databases}
assert set(by_name) == {"default", "test_namespace", "top"}
top_schemas = {s.name: s for s in by_name["top"].schemas}
# Sub-namespace is present but its tables/children are deferred.
assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
assert top_schemas["nested"].tables_resolved is False
assert top_schemas["nested"].child_schemas_resolved is False
assert top_schemas["nested"].tables == []
assert top_schemas["nested"].child_schemas == []
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_databases_lazy_schemas(memory_catalog: Catalog) -> None:
"""With include_schemas=False, databases defer schema discovery."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
databases = engine.get_databases(
include_schemas=False,
include_tables=False,
include_table_details=False,
)
assert {db.name for db in databases} == {
"default",
"test_namespace",
"top",
}
for db in databases:
assert db.schemas_resolved is False
assert db.schemas == []
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_connection_is_lazy(memory_catalog: Catalog) -> None:
"""The initial connection lists top-level namespaces and the first level of
sub-namespaces, but defers their tables and deeper namespaces until expand."""
from marimo._sql.get_engines import engine_to_data_source_connection
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
# The first level of schemas is eager; tables/columns are deferred.
assert engine.inference_config.auto_discover_schemas is True
assert engine.inference_config.auto_discover_tables is False
assert engine.inference_config.auto_discover_columns is False
connection = engine_to_data_source_connection(
VariableName("my_iceberg"), engine
)
by_name = {db.name: db for db in connection.databases}
assert set(by_name) == {"default", "test_namespace", "top"}
# First-level schemas are resolved...
top = by_name["top"]
assert top.schemas_resolved is True
top_schemas = {s.name: s for s in top.schemas}
assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
# ...but their tables and deeper sub-namespaces are deferred.
assert top_schemas[NO_SCHEMA_NAME].tables_resolved is False
assert top_schemas[NO_SCHEMA_NAME].tables == []
assert top_schemas["nested"].tables_resolved is False
assert top_schemas["nested"].child_schemas_resolved is False
assert top_schemas["nested"].tables == []
assert top_schemas["nested"].child_schemas == []
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_schemas_by_path(memory_catalog: Catalog) -> None:
"""get_schemas lists one level at a time, selected by schema_path."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
# Top level: the schemaless entry plus the immediate child "nested"
# (deferred, not recursed).
schemas = engine.get_schemas(
database="top",
include_tables=False,
include_table_details=False,
schema_path=[],
)
assert [s.name for s in schemas] == [NO_SCHEMA_NAME, "nested"]
nested = schemas[1]
assert nested.child_schemas_resolved is False
assert nested.tables_resolved is False
assert nested.child_schemas == []
# Immediate child of "top.nested" is "deep".
schemas = engine.get_schemas(
database="top",
include_tables=False,
include_table_details=False,
schema_path=["nested"],
)
assert [s.name for s in schemas] == ["deep"]
# "top.nested.deep" is a leaf.
assert (
engine.get_schemas(
database="top",
include_tables=False,
include_table_details=False,
schema_path=["nested", "deep"],
)
== []
)
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_nested_namespace_tables(memory_catalog: Catalog) -> None:
"""Tables of a nested namespace are reachable via its dotted name."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
tables = engine.get_tables_in_schema(
schema=NO_SCHEMA_NAME,
database="top.nested",
include_table_details=True,
)
assert [t.name for t in tables] == ["table4"]
assert tables[0].num_columns == 3
table = engine.get_table_details(
table_name="table5",
schema_name=NO_SCHEMA_NAME,
database_name="top.nested.deep",
)
assert table is not None
assert table.name == "table5"
assert len(table.columns) == 3
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_table_calls_fold_schema_path(
memory_catalog: Catalog,
) -> None:
"""The handler passes the top-level `database` plus a `schema_path`; the
engine folds them into a dotted namespace internally (this replaced the
handler-side `_table_database` helper)."""
engine = PyIcebergEngine(
memory_catalog, engine_name=VariableName("my_iceberg")
)
# database + schema_path is equivalent to the pre-folded dotted database.
tables = engine.get_tables_in_schema(
schema=NO_SCHEMA_NAME,
database="top",
schema_path=["nested"],
include_table_details=True,
)
assert [t.name for t in tables] == ["table4"]
table = engine.get_table_details(
table_name="table5",
schema_name=NO_SCHEMA_NAME,
database_name="top",
schema_path=["nested", "deep"],
)
assert table is not None
assert table.name == "table5"
# Empty / missing schema_path leaves the database untouched.
assert PyIcebergEngine._qualified_namespace("top", []) == "top"
assert PyIcebergEngine._qualified_namespace("top", None) == "top"
assert (
PyIcebergEngine._qualified_namespace("top", ["nested", "deep"])
== "top.nested.deep"
)
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_auto_discovery(memory_catalog: Catalog) -> None:
"""Test PyIcebergEngine auto discovery behavior."""
engine_name = VariableName("my_iceberg")
engine = PyIcebergEngine(memory_catalog, engine_name=engine_name)
# Test with auto parameters (should behave like True for PyIceberg)
databases = engine.get_databases(
include_schemas="auto",
include_tables="auto",
include_table_details="auto",
)
assert isinstance(databases, list)
assert len(databases) == 3
by_name = {db.name: db for db in databases}
default_schemas = {s.name: s for s in by_name["default"].schemas}
assert len(default_schemas[NO_SCHEMA_NAME].tables) == 2
# Test with _is_cheap_discovery mocked to return False
with mock.patch.object(
PyIcebergEngine, "_is_cheap_discovery", return_value=False
):
databases = engine.get_databases(
include_schemas="auto",
include_tables="auto",
include_table_details="auto",
)
assert isinstance(databases, list)
assert len(databases) == 3
for db in databases:
assert db.schemas_resolved is False
assert db.schemas == []