Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

gate schema discovery behind "auto" for remote engines (#9784)

Shahmir Varqha981108bed5ce
diff --git a/marimo/_sql/engines/adbc.py b/marimo/_sql/engines/adbc.py
index c063dff4fd..de4b3bcd87 100644
--- a/marimo/_sql/engines/adbc.py
+++ b/marimo/_sql/engines/adbc.py
@@ -13,8 +13,12 @@ from marimo._data.models import (
DataType,
Schema,
)
-from marimo._sql.engines.types import InferenceConfig, SQLConnection
-from marimo._sql.utils import CHEAP_DISCOVERY_DATABASES, convert_to_output
+from marimo._sql.engines.types import (
+ InferenceConfig,
+ SQLConnection,
+ default_inference_config,
+)
+from marimo._sql.utils import convert_to_output, is_cheap_dialect
from marimo._types.ids import VariableName
LOGGER = _loggers.marimo_logger()
@@ -189,7 +193,7 @@ class AdbcConnectionCatalog:
self, value: bool | Literal["auto"]
) -> bool:
if value == "auto":
- return self._dialect.lower() in CHEAP_DISCOVERY_DATABASES
+ return is_cheap_dialect(self._dialect)
return value
def get_databases(
@@ -489,11 +493,7 @@ class AdbcDBAPIEngine(SQLConnection[AdbcDbApiConnection]):
@property
def inference_config(self) -> InferenceConfig:
- return InferenceConfig(
- auto_discover_schemas=True,
- auto_discover_tables="auto",
- auto_discover_columns=False,
- )
+ return default_inference_config()
def get_default_database(self) -> str | None:
return self._catalog.get_default_database()
diff --git a/marimo/_sql/engines/clickhouse.py b/marimo/_sql/engines/clickhouse.py
index ddba96eb04..259b8b328d 100644
--- a/marimo/_sql/engines/clickhouse.py
+++ b/marimo/_sql/engines/clickhouse.py
@@ -371,13 +371,9 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
)
return databases
- def _resolve_should_auto_discover(
- self, value: bool | Literal["auto"]
- ) -> bool:
- if value == "auto":
- # TODO: Smartly determine if we should auto-discover
- return False
- return value
+ def _is_cheap_discovery(self) -> bool:
+ # TODO: Smartly determine if we should auto-discover
+ return False
def get_tables_in_schema(
self,
diff --git a/marimo/_sql/engines/ibis.py b/marimo/_sql/engines/ibis.py
index 9a9fbd38ee..32ef77fbf8 100644
--- a/marimo/_sql/engines/ibis.py
+++ b/marimo/_sql/engines/ibis.py
@@ -12,8 +12,12 @@ from marimo._data.models import (
Schema,
)
from marimo._dependencies.dependencies import DependencyManager
-from marimo._sql.engines.types import InferenceConfig, SQLConnection
-from marimo._sql.utils import CHEAP_DISCOVERY_DATABASES, convert_to_output
+from marimo._sql.engines.types import (
+ InferenceConfig,
+ SQLConnection,
+ default_inference_config,
+)
+from marimo._sql.utils import convert_to_output
from marimo._types.ids import VariableName
if TYPE_CHECKING:
@@ -84,11 +88,7 @@ class IbisEngine(SQLConnection["SQLBackend"]):
@property
def inference_config(self) -> InferenceConfig:
- return InferenceConfig(
- auto_discover_schemas=True,
- auto_discover_tables="auto",
- auto_discover_columns=False,
- )
+ return default_inference_config()
def get_default_database(self) -> str | None:
"""Get the current database name.
@@ -460,14 +460,3 @@ class IbisEngine(SQLConnection["SQLBackend"]):
return "string"
else:
raise IbisToMarimoConversionError
-
- def _resolve_should_auto_discover(
- self,
- value: bool | Literal["auto"],
- ) -> bool:
- if value == "auto":
- return self._is_cheap_discovery()
- return value
-
- def _is_cheap_discovery(self) -> bool:
- return self.dialect.lower() in CHEAP_DISCOVERY_DATABASES
diff --git a/marimo/_sql/engines/pyiceberg.py b/marimo/_sql/engines/pyiceberg.py
index 28be5456bb..e1de4646f1 100644
--- a/marimo/_sql/engines/pyiceberg.py
+++ b/marimo/_sql/engines/pyiceberg.py
@@ -216,13 +216,5 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
)
return None
- def _resolve_should_auto_discover(
- self,
- value: bool | Literal["auto"],
- ) -> bool:
- if value == "auto":
- return self._is_cheap_discovery()
- return value
-
def _is_cheap_discovery(self) -> bool:
return True # Iceberg metadata is generally fast to access
diff --git a/marimo/_sql/engines/redshift.py b/marimo/_sql/engines/redshift.py
index 4de001f835..36a8b42eb4 100644
--- a/marimo/_sql/engines/redshift.py
+++ b/marimo/_sql/engines/redshift.py
@@ -420,10 +420,6 @@ class RedshiftEngine(SQLConnection["Connection"]):
return "string"
return None
- def _resolve_should_auto_discover(
- self, value: bool | Literal["auto"]
- ) -> bool:
+ def _is_cheap_discovery(self) -> bool:
# Opt to not auto-discover for now
- if value == "auto":
- return False
- return value
+ return False
diff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.py
index 4a94105383..a69159349f 100644
--- a/marimo/_sql/engines/sqlalchemy.py
+++ b/marimo/_sql/engines/sqlalchemy.py
@@ -22,9 +22,12 @@ from marimo._data.models import (
Schema,
)
from marimo._dependencies.dependencies import DependencyManager
-from marimo._sql.engines.types import InferenceConfig, SQLConnection
+from marimo._sql.engines.types import (
+ InferenceConfig,
+ SQLConnection,
+ default_inference_config,
+)
from marimo._sql.utils import (
- CHEAP_DISCOVERY_DATABASES,
convert_to_output,
sql_type_to_data_type,
)
@@ -224,11 +227,7 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
@property
def inference_config(self) -> InferenceConfig:
- return InferenceConfig(
- auto_discover_schemas="auto",
- auto_discover_tables="auto",
- auto_discover_columns=False,
- )
+ return default_inference_config()
def get_default_database(self) -> str | None:
"""Get the current database name.
@@ -691,17 +690,6 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
col_type = engine_type.as_generic()
return sql_type_to_data_type(str(col_type))
- def _resolve_should_auto_discover(
- self,
- value: bool | Literal["auto"],
- ) -> bool:
- if value == "auto":
- return self._is_cheap_discovery()
- return value
-
- def _is_cheap_discovery(self) -> bool:
- return self.dialect.lower() in CHEAP_DISCOVERY_DATABASES
-
@staticmethod
def is_cursor_result(result: Any) -> bool:
if not DependencyManager.sqlalchemy.has():
diff --git a/marimo/_sql/engines/types.py b/marimo/_sql/engines/types.py
index d62f1e61a8..0d5e3833cb 100644
--- a/marimo/_sql/engines/types.py
+++ b/marimo/_sql/engines/types.py
@@ -14,6 +14,7 @@ from marimo._sql.parse import (
)
from marimo._sql.utils import (
get_configured_sql_output_format,
+ is_cheap_dialect,
is_query_empty,
strip_explain_from_error_message,
wrap_query_with_explain,
@@ -30,6 +31,19 @@ class InferenceConfig(ABC):
auto_discover_columns: bool | Literal["auto"]
+def default_inference_config() -> InferenceConfig:
+ """Default discovery config shared by general-purpose SQL engines.
+
+ Expensive backends can have a large number of schemas and tables, so we
+ gate discovery behind the `"auto"` heuristic.
+ """
+ return InferenceConfig(
+ auto_discover_schemas="auto",
+ auto_discover_tables="auto",
+ auto_discover_columns=False,
+ )
+
+
def _validate_sql_output_format(sql_output: SqlOutputType) -> SqlOutputType:
if sql_output in ("lazy-polars", "polars"):
DependencyManager.polars.require(
@@ -121,6 +135,23 @@ class EngineCatalog(BaseEngine[CONN], ABC):
) -> DataTable | None:
"""Get a single table from the engine."""
+ def _resolve_should_auto_discover(
+ self, value: bool | Literal["auto"]
+ ) -> bool:
+ """Resolve a discovery flag, deferring `"auto"` to engine policy."""
+ if value == "auto":
+ return self._is_cheap_discovery()
+ return value
+
+ def _is_cheap_discovery(self) -> bool:
+ """Whether discovery is cheap enough to run when a flag is `"auto"`.
+
+ Defaults to a dialect-based heuristic; engines with different cost
+ profiles (e.g. always-cheap local catalogs, or expensive remote
+ warehouses) should override this.
+ """
+ return is_cheap_dialect(self.dialect)
+
class QueryEngine(BaseEngine[CONN], ABC):
"""Protocol for SQL engines that can execute queries."""
diff --git a/marimo/_sql/utils.py b/marimo/_sql/utils.py
index d98802a2ae..dadc778357 100644
--- a/marimo/_sql/utils.py
+++ b/marimo/_sql/utils.py
@@ -27,11 +27,28 @@ if TYPE_CHECKING:
LOGGER = _loggers.marimo_logger()
-CHEAP_DISCOVERY_DATABASES = ["duckdb", "sqlite", "mysql", "postgresql"]
+CHEAP_DISCOVERY_DATABASES = {
+ "duckdb",
+ "sqlite",
+ "mysql",
+ "mariadb",
+ "postgresql",
+}
# DuckDB SQL can return None for DDL, so keep "patch did not apply" distinct.
_NO_WASM_DUCKDB_RESULT = object()
+def is_cheap_dialect(dialect: str) -> bool:
+ """Whether schema/table discovery is cheap for the given SQL dialect.
+
+ Used as the default heuristic for resolving `"auto"` discovery flags: local
+ or lightweight dialects are cheap to introspect, whereas remote analytical
+ backends (e.g. Snowflake, BigQuery) can be slow and should not be scanned
+ eagerly.
+ """
+ return dialect.lower() in CHEAP_DISCOVERY_DATABASES
+
+
def get_configured_sql_output_format() -> SqlOutputType:
"""Read the configured SQL output format from the runtime context.
diff --git a/tests/_sql/test_adbc.py b/tests/_sql/test_adbc.py
index 1b2bb191ce..7e6c281239 100644
--- a/tests/_sql/test_adbc.py
+++ b/tests/_sql/test_adbc.py
@@ -13,6 +13,7 @@ from marimo._sql.engines.adbc import (
_adbc_info_to_dialect,
_schema_field_to_data_type,
)
+from marimo._sql.engines.types import default_inference_config
from marimo._sql.get_engines import get_engines_from_variables
from marimo._types.ids import VariableName
@@ -438,6 +439,18 @@ def test_adbc_is_compatible_does_not_create_cursor() -> None:
assert conn._cursor.did_close is True # type: ignore[attr-defined]
+def test_adbc_engine_uses_default_inference_config() -> None:
+ """ADBC shares the default discovery config (see #9775)."""
+ conn = FakeAdbcDbApiConnection(
+ cursor=FakeAdbcDbApiCursor(description=None),
+ objects_pylist=[],
+ table_schema=FakeAdbcTableSchema([]),
+ )
+ engine = AdbcDBAPIEngine(conn)
+ assert engine.inference_config == default_inference_config()
+ assert engine.inference_config.auto_discover_schemas == "auto"
+
+
def test_adbc_catalog_auto_discovery_uses_cheap_dialect_heuristic() -> None:
conn = FakeAdbcDbApiConnection(
cursor=FakeAdbcDbApiCursor(description=None),
diff --git a/tests/_sql/test_clickhouse.py b/tests/_sql/test_clickhouse.py
index 832350a603..2cce5d74ec 100644
--- a/tests/_sql/test_clickhouse.py
+++ b/tests/_sql/test_clickhouse.py
@@ -99,6 +99,27 @@ def test_clickhouse_server_creation() -> None:
assert connection.source == "clickhouse"
[email protected](not HAS_PANDAS, reason="pandas not installed")
+def test_clickhouse_server_get_databases_auto_skips_tables() -> None:
+ """ClickHouse server is not cheap, so `"auto"` must not scan tables."""
+ import pandas as pd
+
+ connection = mock.MagicMock()
+ connection.query_df.return_value = pd.DataFrame({"name": ["db1", "db2"]})
+
+ engine = ClickhouseServer(connection)
+ with mock.patch.object(engine, "get_tables_in_schema") as mock_get_tables:
+ databases = engine.get_databases(
+ include_schemas="auto",
+ include_tables="auto",
+ include_table_details="auto",
+ )
+
+ mock_get_tables.assert_not_called()
+ assert [db.name for db in databases] == ["db1", "db2"]
+ assert all(db.schemas[0].tables == [] for db in databases)
+
+
@pytest.mark.skipif(
not HAS_CHDB and not HAS_POLARS and not HAS_PANDAS,
reason="chdb, polars, and pandas not installed",
diff --git a/tests/_sql/test_ibis.py b/tests/_sql/test_ibis.py
index db1fa11569..4b2b44f380 100644
--- a/tests/_sql/test_ibis.py
+++ b/tests/_sql/test_ibis.py
@@ -10,7 +10,11 @@ import pytest
from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._dependencies.dependencies import DependencyManager
from marimo._sql.engines.ibis import IbisEngine, IbisToMarimoConversionError
-from marimo._sql.engines.types import EngineCatalog, QueryEngine
+from marimo._sql.engines.types import (
+ EngineCatalog,
+ QueryEngine,
+ default_inference_config,
+)
from marimo._sql.sql import sql
from marimo._types.ids import VariableName
@@ -506,6 +510,16 @@ def test_ibis_engine_get_schemas(ibis_backend: SQLBackend) -> None:
assert len(schema.tables) == 0
[email protected](not HAS_IBIS, reason="Ibis not installed")
+def test_ibis_engine_uses_default_inference_config(
+ ibis_backend: SQLBackend,
+) -> None:
+ """Ibis shares the default discovery config (see #9775)."""
+ engine = IbisEngine(ibis_backend, engine_name=VariableName("my_ibis"))
+ assert engine.inference_config == default_inference_config()
+ assert engine.inference_config.auto_discover_schemas == "auto"
+
+
@pytest.mark.skipif(not HAS_IBIS, reason="Ibis not installed")
def test_ibis_engine_get_databases(ibis_backend: SQLBackend) -> None:
"""Test IbisEngine get_databases method."""
diff --git a/tests/_sql/test_sqlalchemy.py b/tests/_sql/test_sqlalchemy.py
index 188517a9f4..c139354376 100644
--- a/tests/_sql/test_sqlalchemy.py
+++ b/tests/_sql/test_sqlalchemy.py
@@ -11,7 +11,11 @@ import pytest
from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._dependencies.dependencies import DependencyManager
from marimo._sql.engines.sqlalchemy import SQLAlchemyEngine, safe_execute
-from marimo._sql.engines.types import EngineCatalog, QueryEngine
+from marimo._sql.engines.types import (
+ EngineCatalog,
+ QueryEngine,
+ default_inference_config,
+)
from marimo._sql.sql import sql
from marimo._types.ids import VariableName
@@ -115,6 +119,17 @@ def test_sqlalchemy_engine_dialect(sqlite_engine: sa.Engine) -> None:
assert engine.dialect == "sqlite"
[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
+def test_sqlalchemy_engine_uses_default_inference_config(
+ sqlite_engine: sa.Engine,
+) -> None:
+ """SQLAlchemy shares the default discovery config (see #9775)."""
+ engine = SQLAlchemyEngine(
+ sqlite_engine, engine_name=VariableName("test_sqlite")
+ )
+ assert engine.inference_config == default_inference_config()
+
+
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
def test_sqlalchemy_engine_is_instance(sqlite_engine: sa.Engine) -> None:
"""Test SQLAlchemyEngine is an instance of the correct types."""