Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

perf(sql): make catalog discovery reuse one connection (#10636)

Shahmir Varqha140bf820e713
diff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.py
index 89f8e987da..1bfe6abf81 100644
--- a/marimo/_sql/engines/sqlalchemy.py
+++ b/marimo/_sql/engines/sqlalchemy.py
@@ -94,6 +94,13 @@ def safe_execute(
return decorator
+def _index_by_table_name(
+ reflected: dict[tuple[str | None, str], T],
+) -> dict[str, T]:
+ """Index reflection results by table name within a single schema."""
+ return {name: value for (_schema, name), value in reflected.items()}
+
+
# ------------------------------------------------------------------ #
# SQLAlchemyEngine #
# ------------------------------------------------------------------ #
@@ -507,22 +514,19 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
# Tables resolution #
# -------------------------------------------------------------- #
+ @staticmethod
+ def _get_table_names(
+ inspector: Inspector, schema: str
+ ) -> tuple[list[str], list[str]]:
+ return inspector.get_table_names(
+ schema=schema
+ ), inspector.get_view_names(schema=schema)
+
@safe_execute(
- fallback=([], []),
+ fallback=[],
message="Failed to get tables in schema",
log_level="warning",
)
- def _get_table_names(
- self, schema: str, database: str
- ) -> tuple[list[str], list[str]]:
-
- with self._get_inspector(database) as inspector:
- if inspector is None:
- return [], []
- return inspector.get_table_names(
- schema=schema
- ), inspector.get_view_names(schema=schema)
-
def get_tables_in_schema(
self,
*,
@@ -534,15 +538,30 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
"""Return all tables in a schema."""
del schema_path # SQLAlchemy schemas don't nest
- table_names, view_names = self._get_table_names(
- schema=schema, database=database
- )
+ with self._get_inspector(database) as inspector:
+ if inspector is None:
+ return []
+ return self._get_tables_from_inspector(
+ inspector,
+ schema=schema,
+ include_table_details=include_table_details,
+ )
+
+ def _get_tables_from_inspector(
+ self,
+ inspector: Inspector,
+ *,
+ schema: str,
+ include_table_details: bool,
+ ) -> list[DataTable]:
+ table_names, view_names = self._get_table_names(inspector, schema)
+ tables: list[tuple[DataTableType, str]] = [
+ ("table", name) for name in table_names
+ ]
+ tables.extend(("view", name) for name in view_names)
- tables: list[tuple[DataTableType, str]] = []
- for name in table_names:
- tables.append(("table", name))
- for name in view_names:
- tables.append(("view", name))
+ if not tables:
+ return []
if not include_table_details:
return [
@@ -562,13 +581,35 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
for table_type, name in tables
]
- data_tables: list[DataTable] = []
- for t_type, t_name in tables:
- table = self.get_table_details(
- table_name=t_name, schema_name=schema, database_name=database
+ batched = self._get_batched_table_details(
+ inspector,
+ table_names=table_names,
+ view_names=view_names,
+ schema=schema,
+ )
+ if batched is None:
+ batched = {}
+ elif len(batched) < len(tables):
+ LOGGER.debug(
+ "Batched reflection returned %d of %d tables in schema %s",
+ len(batched),
+ len(tables),
+ schema,
)
+
+ data_tables: list[DataTable] = []
+ for table_type, table_name in tables:
+ # Tables missing from the batch (stale listings, dialect quirks)
+ # fall back on the same inspector and connection.
+ table = batched.get(table_name)
+ if table is None:
+ table = self._get_table_details_from_inspector(
+ inspector,
+ table_name=table_name,
+ schema=schema,
+ )
if table is not None:
- table.type = t_type
+ table.type = table_type
data_tables.append(table)
return data_tables
@@ -577,42 +618,36 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
# Table Details resolution #
# -------------------------------------------------------------- #
+ @staticmethod
@safe_execute(
fallback=None,
message="Failed to get table details",
log_level="warning",
)
def _get_columns(
- self, table_name: str, schema: str, database: str
+ inspector: Inspector, *, table_name: str, schema: str
) -> list[ReflectedColumn] | None:
+ return inspector.get_columns(table_name, schema=schema)
- with self._get_inspector(database) as inspector:
- if inspector is None:
- return None
- return inspector.get_columns(table_name, schema=schema)
-
+ @staticmethod
@safe_execute(fallback=[], message="Failed to get primary keys")
def _fetch_primary_keys(
- self, table_name: str, schema: str, database: str
+ inspector: Inspector, *, table_name: str, schema: str
) -> list[str]:
-
- with self._get_inspector(database) as inspector:
- if inspector is None:
- return []
- return inspector.get_pk_constraint(table_name, schema=schema).get(
- "constrained_columns", []
- )
+ return inspector.get_pk_constraint(table_name, schema=schema).get(
+ "constrained_columns", []
+ )
@safe_execute(fallback=[], message="Failed to get indexes")
def _fetch_indexes(
- self, table_name: str, schema: str, database: str
+ self,
+ inspector: Inspector,
+ *,
+ table_name: str,
+ schema: str,
) -> list[str]:
-
- with self._get_inspector(database) as inspector:
- if inspector is None:
- return []
- indexes = inspector.get_indexes(table_name, schema=schema)
- return self._extract_index_columns(indexes)
+ indexes = inspector.get_indexes(table_name, schema=schema)
+ return self._extract_index_columns(indexes)
@staticmethod
def _extract_index_columns(indexes: list[ReflectedIndex]) -> list[str]:
@@ -627,6 +662,120 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
index_columns.append(col)
return index_columns
+ @safe_execute(
+ fallback=None,
+ message="Failed to get batched table details",
+ log_level="warning",
+ )
+ def _get_batched_table_details(
+ self,
+ inspector: Inspector,
+ *,
+ table_names: list[str],
+ view_names: list[str],
+ schema: str,
+ ) -> dict[str, DataTable] | None:
+ """Reflect details for many tables at once."""
+ if not hasattr(inspector, "get_multi_columns"):
+ return None
+
+ # Only importable on SQLAlchemy 2.0+, hence the hasattr guard
+ from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
+
+ object_names = [*table_names, *view_names]
+ try:
+ multi_columns = inspector.get_multi_columns(
+ schema=schema,
+ filter_names=object_names,
+ kind=ObjectKind.ANY,
+ scope=ObjectScope.ANY,
+ )
+ except Exception:
+ LOGGER.warning("Failed to get batched columns", exc_info=True)
+ return None
+
+ try:
+ multi_pk_constraints = (
+ inspector.get_multi_pk_constraint(
+ schema=schema,
+ filter_names=table_names,
+ kind=ObjectKind.ANY,
+ scope=ObjectScope.ANY,
+ )
+ if table_names
+ else {}
+ )
+ except Exception:
+ LOGGER.warning("Failed to get batched primary keys", exc_info=True)
+ multi_pk_constraints = None
+
+ try:
+ multi_indexes = (
+ inspector.get_multi_indexes(
+ schema=schema,
+ filter_names=table_names,
+ kind=ObjectKind.ANY,
+ scope=ObjectScope.ANY,
+ )
+ if table_names
+ else {}
+ )
+ except Exception:
+ LOGGER.warning("Failed to get batched indexes", exc_info=True)
+ multi_indexes = None
+
+ columns_by_table = _index_by_table_name(multi_columns)
+ pk_constraints_by_table = (
+ _index_by_table_name(multi_pk_constraints)
+ if multi_pk_constraints is not None
+ else None
+ )
+ indexes_by_table = (
+ _index_by_table_name(multi_indexes)
+ if multi_indexes is not None
+ else None
+ )
+
+ tables: dict[str, DataTable] = {}
+ table_name_set = set(table_names)
+ for name, columns in columns_by_table.items():
+ if name not in table_name_set:
+ primary_keys = []
+ elif pk_constraints_by_table is None:
+ primary_keys = self._fetch_primary_keys(
+ inspector, table_name=name, schema=schema
+ )
+ else:
+ pk_constraint = pk_constraints_by_table.get(name)
+ primary_keys = (
+ pk_constraint.get("constrained_columns", [])
+ if pk_constraint
+ else []
+ )
+
+ if name not in table_name_set:
+ index_columns = []
+ elif indexes_by_table is None:
+ index_columns = self._fetch_indexes(
+ inspector, table_name=name, schema=schema
+ )
+ else:
+ index_columns = self._extract_index_columns(
+ indexes_by_table.get(name, [])
+ )
+ tables[name] = self._build_data_table(
+ table_name=name,
+ columns=columns,
+ primary_keys=primary_keys,
+ index_list=index_columns,
+ )
+ return tables
+
+ @safe_execute(
+ fallback=None,
+ message="Failed to get table details",
+ log_level="warning",
+ )
def get_table_details(
self,
*,
@@ -638,19 +787,54 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
"""Get a single table from the engine."""
del schema_path # SQLAlchemy schemas don't nest
+ with self._get_inspector(database_name) as inspector:
+ if inspector is None:
+ return None
+ return self._get_table_details_from_inspector(
+ inspector, table_name=table_name, schema=schema_name
+ )
+
+ @safe_execute(
+ fallback=None,
+ message="Failed to get table details",
+ log_level="warning",
+ )
+ def _get_table_details_from_inspector(
+ self,
+ inspector: Inspector,
+ *,
+ table_name: str,
+ schema: str,
+ ) -> DataTable | None:
columns = self._get_columns(
- table_name, schema=schema_name, database=database_name
+ inspector, table_name=table_name, schema=schema
)
if columns is None:
return None
primary_keys = self._fetch_primary_keys(
- table_name, schema_name, database_name
+ inspector, table_name=table_name, schema=schema
)
index_list = self._fetch_indexes(
- table_name, schema_name, database_name
+ inspector, table_name=table_name, schema=schema
+ )
+
+ return self._build_data_table(
+ table_name=table_name,
+ columns=columns,
+ primary_keys=primary_keys,
+ index_list=index_list,
)
+ def _build_data_table(
+ self,
+ *,
+ table_name: str,
+ columns: list[ReflectedColumn],
+ primary_keys: list[str],
+ index_list: list[str],
+ ) -> DataTable:
+ """Convert reflected column metadata into a DataTable."""
cols: list[DataTableColumn] = []
for col in columns:
engine_type = col["type"]
diff --git a/tests/_sql/test_sqlalchemy.py b/tests/_sql/test_sqlalchemy.py
index 9f5479c6e6..5617a10db3 100644
--- a/tests/_sql/test_sqlalchemy.py
+++ b/tests/_sql/test_sqlalchemy.py
@@ -10,7 +10,11 @@ import pytest
from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._dependencies.dependencies import DependencyManager
-from marimo._sql.engines.sqlalchemy import SQLAlchemyEngine, safe_execute
+from marimo._sql.engines.sqlalchemy import (
+ SQLAlchemyEngine,
+ _index_by_table_name,
+ safe_execute,
+)
from marimo._sql.engines.types import (
EngineCatalog,
QueryEngine,
@@ -110,6 +114,50 @@ def sqlite_engine_meta() -> sa.Engine:
return engine
+def sqlite_engine_with_view() -> sa.Engine:
+ """Create a SQLite database with tables and views in two schemas."""
+
+ import sqlalchemy as sa
+ from sqlalchemy import text
+
+ engine = sa.create_engine("sqlite:///:memory:")
+
+ with engine.begin() as conn:
+ conn.execute(
+ text(
+ """
+ CREATE TABLE test (
+ id INTEGER PRIMARY KEY,
+ name TEXT
+ )
+ """
+ )
+ )
+ conn.execute(
+ text("CREATE VIEW test_view AS SELECT id, name FROM test")
+ )
+ conn.execute(text("ATTACH ':memory:' AS my_schema"))
+ conn.execute(
+ text(
+ """
+ CREATE TABLE my_schema.test2 (
+ id INTEGER PRIMARY KEY,
+ name TEXT
+ )
+ """
+ )
+ )
+ conn.execute(
+ text(
+ "CREATE VIEW my_schema.test2_view AS "
+ "SELECT id, name FROM my_schema.test2"
+ )
+ )
+
+ return engine
+
+
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
def test_sqlalchemy_engine_dialect(sqlite_engine: sa.Engine) -> None:
"""Test SQLAlchemyEngine dialect property."""
@@ -304,6 +352,14 @@ def get_expected_schema(schema_name: str, table_name: str) -> Schema:
)
+def get_expected_view(view_name: str) -> DataTable:
+ "Expected reflection of a view over a `get_expected_table` table."
+ view = get_expected_table(view_name)
+ view.type = "view"
+ view.primary_keys = []
+ return view
+
+
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
def test_sqlalchemy_engine_get_table_details(sqlite_engine: sa.Engine) -> None:
"""Test SQLAlchemyEngine get_table method."""
@@ -388,6 +444,120 @@ def test_sqlalchemy_engine_get_tables_in_schema(
assert tables[0] == expected_table
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_sqlalchemy_engine_get_tables_with_views(
+ sqlite_engine_with_view: sa.Engine,
+) -> None:
+ """Views keep type='view' and full details on the batched path."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine_with_view, engine_name=VariableName("test_sqlite")
+ )
+ tables = engine.get_tables_in_schema(
+ schema="main", database=UNUSED_DB_NAME, include_table_details=True
+ )
+ assert tables == [
+ get_expected_table("test"),
+ get_expected_view("test_view"),
+ ]
+
+ tables = engine.get_tables_in_schema(
+ schema="my_schema",
+ database=UNUSED_DB_NAME,
+ include_table_details=True,
+ )
+ assert tables == [
+ get_expected_table("test2"),
+ get_expected_view("test2_view"),
+ ]
+
+
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_batched_details_match_per_table_details(
+ sqlite_engine_with_view: sa.Engine,
+) -> None:
+ """Batched reflection returns the same tables as per-table reflection."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine_with_view, engine_name=VariableName("test_sqlite")
+ )
+ batched = engine.get_tables_in_schema(
+ schema="main", database=UNUSED_DB_NAME, include_table_details=True
+ )
+ with mock.patch.object(
+ engine, "_get_batched_table_details", return_value=None
+ ):
+ per_table = engine.get_tables_in_schema(
+ schema="main",
+ database=UNUSED_DB_NAME,
+ include_table_details=True,
+ )
+ assert batched == per_table
+
+
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_batched_details_avoid_per_table_reflection(
+ sqlite_engine: sa.Engine,
+) -> None:
+ """No per-table reflection runs when the batch covers every table."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine, engine_name=VariableName("test_sqlite")
+ )
+ with mock.patch.object(
+ engine, "_get_columns", wraps=engine._get_columns
+ ) as spy:
+ tables = engine.get_tables_in_schema(
+ schema="main",
+ database=UNUSED_DB_NAME,
+ include_table_details=True,
+ )
+ assert tables == [get_expected_table("test")]
+ spy.assert_not_called()
+
+
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_batched_details_fall_back_when_multi_reflection_raises(
+ sqlite_engine: sa.Engine,
+) -> None:
+ """A failing batch logs one warning and falls back per table."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine, engine_name=VariableName("test_sqlite")
+ )
+ with (
+ mock.patch.object(
+ engine.inspector,
+ "get_multi_columns",
+ side_effect=RuntimeError("boom"),
+ ),
+ mock.patch("marimo._sql.engines.sqlalchemy.LOGGER") as mock_logger,
+ ):
+ tables = engine.get_tables_in_schema(
+ schema="main",
+ database=UNUSED_DB_NAME,
+ include_table_details=True,
+ )
+ assert tables == [get_expected_table("test")]
+ mock_logger.warning.assert_called_once_with(
+ "Failed to get batched columns", exc_info=True
+ )
+
+
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_nonexistent_schema_skips_batched_reflection(
+ sqlite_engine: sa.Engine,
+) -> None:
+ """A schema with no tables returns [] without attempting a batch."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine, engine_name=VariableName("test_sqlite")
+ )
+ with mock.patch.object(engine, "_get_batched_table_details") as batched:
+ tables = engine.get_tables_in_schema(
+ schema="non_existent",
+ database=UNUSED_DB_NAME,
+ include_table_details=True,
+ )
+ assert tables == []
+ batched.assert_not_called()
+
+
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
def test_sqlalchemy_skip_meta_schemas(
sqlite_engine_meta: sa.Engine,
@@ -874,6 +1044,10 @@ def make_engine():
rows=None,
pk_constraint=None,
indexes=None,
+ multi_columns=None,
+ multi_pk_constraint=None,
+ multi_indexes=None,
+ supports_multi=True,
inspector=True,
):
if columns is None:
@@ -907,6 +1081,23 @@ def make_engine():
"constrained_columns": []
}
mock_inspector.get_indexes.return_value = indexes or []
+ if supports_multi:
+ # Empty dicts by default so tests exercise the per-table
+ # fallback instead of passing on bare MagicMock returns.
+ mock_inspector.get_multi_columns.return_value = (
+ multi_columns or {}
+ )
+ mock_inspector.get_multi_pk_constraint.return_value = (
+ multi_pk_constraint or {}
+ )
+ mock_inspector.get_multi_indexes.return_value = (
+ multi_indexes or {}
+ )
+ else:
+ # Simulate SQLAlchemy < 2.0, where hasattr(...) is False
+ del mock_inspector.get_multi_columns
+ del mock_inspector.get_multi_pk_constraint
+ del mock_inspector.get_multi_indexes
# -- Mock SQLAlchemy engine --
mock_sa_engine = mock.MagicMock()
@@ -930,9 +1121,11 @@ def make_engine():
# Uses *_args, **_kwargs to match any signature the real
# method may have (e.g. database parameter) without
# global side effects.
+ inspector_contexts = []
+
@contextmanager
def fake_get_inspector(database=None):
- _ = database # ignore parameter
+ inspector_contexts.append(database)
yield mock_inspector
engine._get_inspector = fake_get_inspector
@@ -940,6 +1133,7 @@ def make_engine():
# Expose mocks for test assertions
engine._mock_inspector = mock_inspector
engine._mock_conn = mock_conn
+ engine._mock_inspector_contexts = inspector_contexts
return engine
@@ -1028,6 +1222,45 @@ def test_get_inspector_yields_existing_inspector_for_non_use_database_dialect():
mock_sa_engine.connect.assert_not_called()
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_get_tables_in_schema_reuses_connection():
+ import sqlalchemy as sa
+
+ mock_inspector = mock.MagicMock()
+ mock_inspector.get_table_names.return_value = ["users"]
+ mock_inspector.get_view_names.return_value = []
+ mock_inspector.get_multi_columns.return_value = {
+ ("public", "users"): [{"name": "id", "type": sa.INTEGER()}]
+ }
+ mock_inspector.get_multi_pk_constraint.return_value = {
+ ("public", "users"): {"constrained_columns": ["id"]}
+ }
+ mock_inspector.get_multi_indexes.return_value = {("public", "users"): []}
+
+ mock_connection = mock.MagicMock()
+ mock_sa_engine = mock.MagicMock()
+ mock_sa_engine.dialect.name = "snowflake"
+ mock_sa_engine.url.database = "MY_DB"
+ mock_sa_engine.connect.return_value.__enter__.return_value = (
+ mock_connection
+ )
+
+ with mock.patch("sqlalchemy.inspect", return_value=mock_inspector):
+ engine = SQLAlchemyEngine(
+ mock_sa_engine, engine_name=VariableName("test")
+ )
+ tables = engine.get_tables_in_schema(
+ schema="public",
+ database="MY_DB",
+ include_table_details=True,
+ )
+
+ assert [table.name for table in tables] == ["users"]
+ mock_sa_engine.connect.assert_called_once_with()
+ executed = mock_connection.execute.call_args[0][0]
+ assert str(executed) == 'USE DATABASE "MY_DB"'
+
+
# ------------------------------------------------------------------ #
# _get_schema_names
# ------------------------------------------------------------------ #
@@ -1045,101 +1278,192 @@ def test_get_schema_names_no_inspector(make_engine):
assert engine._get_schema_names("MY_DB") == []
-# ------------------------------------------------------------------ #
-# _get_table_names
-# ------------------------------------------------------------------ #
-
-
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_get_table_names(make_engine):
+def test_get_table_details_reuses_inspector_context(make_engine):
+ import sqlalchemy as sa
+
engine = make_engine(
- table_names=["users", "orders"], view_names=["active_users"]
+ table_columns=[{"name": "id", "type": sa.INTEGER()}],
+ pk_constraint={"constrained_columns": ["id"]},
)
- tables, views = engine._get_table_names(schema="public", database="MY_DB")
- assert tables == ["users", "orders"]
- assert views == ["active_users"]
-
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_get_table_names_no_inspector(make_engine):
- engine = make_engine(inspector=False)
- assert engine._get_table_names(schema="public", database="MY_DB") == (
- [],
- [],
+ table = engine.get_table_details(
+ table_name="users",
+ schema_name="public",
+ database_name="MY_DB",
)
+ assert table is not None
+ assert table.primary_keys == ["id"]
+ assert len(engine._mock_inspector_contexts) == 1
+
# ------------------------------------------------------------------ #
-# _get_columns
+# _get_batched_table_details
# ------------------------------------------------------------------ #
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_get_columns(make_engine):
- cols = [
- {"name": "id", "type": "INTEGER"},
- {"name": "name", "type": "TEXT"},
+def test_batched_details_uses_multi_reflection(make_engine):
+ """Details come from get_multi_* calls on a single inspector."""
+ import sqlalchemy as sa
+ from sqlalchemy.engine.reflection import ObjectKind, ObjectScope
+
+ engine = make_engine(
+ table_names=["users"],
+ view_names=["active_users"],
+ multi_columns={
+ ("public", "users"): [{"name": "id", "type": sa.INTEGER()}],
+ ("public", "active_users"): [{"name": "id", "type": sa.INTEGER()}],
+ },
+ multi_pk_constraint={
+ ("public", "users"): {"constrained_columns": ["id"]}
+ },
+ multi_indexes={
+ ("public", "users"): [{"column_names": ["id"], "name": "idx"}]
+ },
+ )
+ tables = engine.get_tables_in_schema(
+ schema="public", database="MY_DB", include_table_details=True
+ )
+
+ assert [(t.name, t.type) for t in tables] == [
+ ("users", "table"),
+ ("active_users", "view"),
]
- engine = make_engine(table_columns=cols)
- result = engine._get_columns("users", schema="public", database="MY_DB")
- assert result == cols
+ assert tables[0].primary_keys == ["id"]
+ assert tables[0].indexes == ["id"]
+ # Missing PK/index entries fall back to [] like per-table reflection
+ assert tables[1].primary_keys == []
+ assert tables[1].indexes == []
+
+ engine._mock_inspector.get_multi_columns.assert_called_once_with(
+ schema="public",
+ filter_names=["users", "active_users"],
+ kind=ObjectKind.ANY,
+ scope=ObjectScope.ANY,
+ )
+ for multi_method in (
+ engine._mock_inspector.get_multi_pk_constraint,
+ engine._mock_inspector.get_multi_indexes,
+ ):
+ multi_method.assert_called_once_with(
+ schema="public",
+ filter_names=["users"],
+ kind=ObjectKind.ANY,
+ scope=ObjectScope.ANY,
+ )
+ engine._mock_inspector.get_columns.assert_not_called()
+ engine._mock_inspector.get_pk_constraint.assert_not_called()
+ engine._mock_inspector.get_indexes.assert_not_called()
+ assert len(engine._mock_inspector_contexts) == 1
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_get_columns_no_inspector(make_engine):
- engine = make_engine(inspector=False)
- assert (
- engine._get_columns("users", schema="public", database="MY_DB") is None
- )
+def test_batched_details_without_multi_support(make_engine):
+ """Inspectors without get_multi_* fall back per table, silently."""
+ import sqlalchemy as sa
+ engine = make_engine(
+ table_names=["users"],
+ table_columns=[{"name": "id", "type": sa.INTEGER()}],
+ pk_constraint={"constrained_columns": ["id"]},
+ supports_multi=False,
+ )
+ with mock.patch("marimo._sql.engines.sqlalchemy.LOGGER") as mock_logger:
+ tables = engine.get_tables_in_schema(
+ schema="public", database="MY_DB", include_table_details=True
+ )
-# ------------------------------------------------------------------ #
-# _fetch_primary_keys
-# ------------------------------------------------------------------ #
+ assert [(t.name, t.type) for t in tables] == [("users", "table")]
+ assert tables[0].primary_keys == ["id"]
+ engine._mock_inspector.get_columns.assert_called_once_with(
+ "users", schema="public"
+ )
+ mock_logger.warning.assert_not_called()
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_fetch_primary_keys(make_engine):
+def test_batched_details_isolates_metadata_failures(make_engine):
+ """Bulk metadata failures reuse batched columns and the same inspector."""
+ import sqlalchemy as sa
+
engine = make_engine(
- pk_constraint={"constrained_columns": ["id", "tenant_id"]}
+ table_names=["users"],
+ view_names=["active_users"],
+ multi_columns={
+ ("public", "users"): [{"name": "id", "type": sa.INTEGER()}],
+ ("public", "active_users"): [{"name": "id", "type": sa.INTEGER()}],
+ },
+ pk_constraint={"constrained_columns": ["id"]},
+ indexes=[{"column_names": ["id"], "name": "idx"}],
+ )
+ engine._mock_inspector.get_multi_pk_constraint.side_effect = RuntimeError
+ engine._mock_inspector.get_multi_indexes.side_effect = RuntimeError
+
+ tables = engine.get_tables_in_schema(
+ schema="public", database="MY_DB", include_table_details=True
+ )
+
+ assert [(table.name, table.type) for table in tables] == [
+ ("users", "table"),
+ ("active_users", "view"),
+ ]
+ assert tables[0].primary_keys == ["id"]
+ assert tables[0].indexes == ["id"]
+ assert tables[1].primary_keys == []
+ assert tables[1].indexes == []
+ engine._mock_inspector.get_columns.assert_not_called()
+ engine._mock_inspector.get_pk_constraint.assert_called_once_with(
+ "users", schema="public"
)
- result = engine._fetch_primary_keys(
- "users", schema="public", database="MY_DB"
+ engine._mock_inspector.get_indexes.assert_called_once_with(
+ "users", schema="public"
)
- assert result == ["id", "tenant_id"]
+ assert len(engine._mock_inspector_contexts) == 1
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_fetch_primary_keys_no_inspector(make_engine):
- engine = make_engine(inspector=False)
- result = engine._fetch_primary_keys(
- "users", schema="public", database="MY_DB"
+def test_batched_details_straggler_falls_back_per_table(make_engine):
+ """A table missing from the batch is fetched per table."""
+ import sqlalchemy as sa
+
+ engine = make_engine(
+ table_names=["users", "orders"],
+ table_columns=[{"name": "id", "type": sa.INTEGER()}],
+ multi_columns={
+ ("public", "users"): [{"name": "id", "type": sa.INTEGER()}]
+ },
)
- assert result == []
+ tables = engine.get_tables_in_schema(
+ schema="public", database="MY_DB", include_table_details=True
+ )
+
+ assert [t.name for t in tables] == ["users", "orders"]
+ engine._mock_inspector.get_columns.assert_called_once_with(
+ "orders", schema="public"
+ )
+ assert len(engine._mock_inspector_contexts) == 1
# ------------------------------------------------------------------ #
-# _fetch_indexes
+# _index_by_table_name (module-level, no mocking needed)
# ------------------------------------------------------------------ #
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_fetch_indexes(make_engine):
- engine = make_engine(
- indexes=[
- {"column_names": ["email"], "name": "idx_email", "unique": True},
- {"column_names": ["name"], "name": "idx_name", "unique": False},
- ]
- )
- result = engine._fetch_indexes("users", schema="public", database="MY_DB")
- assert result == ["email", "name"]
+def test_index_by_table_name():
+ reflected = {
+ ("main", "users"): [{"name": "id"}],
+ (None, "orders"): [{"name": "total"}],
+ }
+ assert _index_by_table_name(reflected) == {
+ "users": [{"name": "id"}],
+ "orders": [{"name": "total"}],
+ }
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
-def test_fetch_indexes_no_inspector(make_engine):
- engine = make_engine(inspector=False)
- result = engine._fetch_indexes("users", schema="public", database="MY_DB")
- assert result == []
+def test_index_by_table_name_empty():
+ assert _index_by_table_name({}) == {}
# ------------------------------------------------------------------ #