A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
gate schema discovery behind "auto" for remote engines (#9784)
Shahmir Varqha981108bed5ce
diff --git a/marimo/_sql/engines/adbc.py b/marimo/_sql/engines/adbc.pyindex c063dff4fd..de4b3bcd87 100644--- a/marimo/_sql/engines/adbc.py+++ b/marimo/_sql/engines/adbc.py@@ -13,8 +13,12 @@ from marimo._data.models import (DataType,Schema,)-from marimo._sql.engines.types import InferenceConfig, SQLConnection-from marimo._sql.utils import CHEAP_DISCOVERY_DATABASES, convert_to_output+from marimo._sql.engines.types import (+ InferenceConfig,+ SQLConnection,+ default_inference_config,+)+from marimo._sql.utils import convert_to_output, is_cheap_dialectfrom marimo._types.ids import VariableNameLOGGER = _loggers.marimo_logger()@@ -189,7 +193,7 @@ class AdbcConnectionCatalog:self, value: bool | Literal["auto"]) -> bool:if value == "auto":- return self._dialect.lower() in CHEAP_DISCOVERY_DATABASES+ return is_cheap_dialect(self._dialect)return valuedef get_databases(@@ -489,11 +493,7 @@ class AdbcDBAPIEngine(SQLConnection[AdbcDbApiConnection]):@propertydef inference_config(self) -> InferenceConfig:- return InferenceConfig(- auto_discover_schemas=True,- auto_discover_tables="auto",- auto_discover_columns=False,- )+ return default_inference_config()def get_default_database(self) -> str | None:return self._catalog.get_default_database()diff --git a/marimo/_sql/engines/clickhouse.py b/marimo/_sql/engines/clickhouse.pyindex ddba96eb04..259b8b328d 100644--- a/marimo/_sql/engines/clickhouse.py+++ b/marimo/_sql/engines/clickhouse.py@@ -371,13 +371,9 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):)return databases- def _resolve_should_auto_discover(- self, value: bool | Literal["auto"]- ) -> bool:- if value == "auto":- # TODO: Smartly determine if we should auto-discover- return False- return value+ def _is_cheap_discovery(self) -> bool:+ # TODO: Smartly determine if we should auto-discover+ return Falsedef get_tables_in_schema(self,diff --git a/marimo/_sql/engines/ibis.py b/marimo/_sql/engines/ibis.pyindex 9a9fbd38ee..32ef77fbf8 100644--- a/marimo/_sql/engines/ibis.py+++ b/marimo/_sql/engines/ibis.py@@ -12,8 +12,12 @@ from marimo._data.models import (Schema,)from marimo._dependencies.dependencies import DependencyManager-from marimo._sql.engines.types import InferenceConfig, SQLConnection-from marimo._sql.utils import CHEAP_DISCOVERY_DATABASES, convert_to_output+from marimo._sql.engines.types import (+ InferenceConfig,+ SQLConnection,+ default_inference_config,+)+from marimo._sql.utils import convert_to_outputfrom marimo._types.ids import VariableNameif TYPE_CHECKING:@@ -84,11 +88,7 @@ class IbisEngine(SQLConnection["SQLBackend"]):@propertydef inference_config(self) -> InferenceConfig:- return InferenceConfig(- auto_discover_schemas=True,- auto_discover_tables="auto",- auto_discover_columns=False,- )+ return default_inference_config()def get_default_database(self) -> str | None:"""Get the current database name.@@ -460,14 +460,3 @@ class IbisEngine(SQLConnection["SQLBackend"]):return "string"else:raise IbisToMarimoConversionError-- def _resolve_should_auto_discover(- self,- value: bool | Literal["auto"],- ) -> bool:- if value == "auto":- return self._is_cheap_discovery()- return value-- def _is_cheap_discovery(self) -> bool:- return self.dialect.lower() in CHEAP_DISCOVERY_DATABASESdiff --git a/marimo/_sql/engines/pyiceberg.py b/marimo/_sql/engines/pyiceberg.pyindex 28be5456bb..e1de4646f1 100644--- a/marimo/_sql/engines/pyiceberg.py+++ b/marimo/_sql/engines/pyiceberg.py@@ -216,13 +216,5 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):)return None- def _resolve_should_auto_discover(- self,- value: bool | Literal["auto"],- ) -> bool:- if value == "auto":- return self._is_cheap_discovery()- return value-def _is_cheap_discovery(self) -> bool:return True # Iceberg metadata is generally fast to accessdiff --git a/marimo/_sql/engines/redshift.py b/marimo/_sql/engines/redshift.pyindex 4de001f835..36a8b42eb4 100644--- a/marimo/_sql/engines/redshift.py+++ b/marimo/_sql/engines/redshift.py@@ -420,10 +420,6 @@ class RedshiftEngine(SQLConnection["Connection"]):return "string"return None- def _resolve_should_auto_discover(- self, value: bool | Literal["auto"]- ) -> bool:+ def _is_cheap_discovery(self) -> bool:# Opt to not auto-discover for now- if value == "auto":- return False- return value+ return Falsediff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.pyindex 4a94105383..a69159349f 100644--- a/marimo/_sql/engines/sqlalchemy.py+++ b/marimo/_sql/engines/sqlalchemy.py@@ -22,9 +22,12 @@ from marimo._data.models import (Schema,)from marimo._dependencies.dependencies import DependencyManager-from marimo._sql.engines.types import InferenceConfig, SQLConnection+from marimo._sql.engines.types import (+ InferenceConfig,+ SQLConnection,+ default_inference_config,+)from marimo._sql.utils import (- CHEAP_DISCOVERY_DATABASES,convert_to_output,sql_type_to_data_type,)@@ -224,11 +227,7 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):@propertydef inference_config(self) -> InferenceConfig:- return InferenceConfig(- auto_discover_schemas="auto",- auto_discover_tables="auto",- auto_discover_columns=False,- )+ return default_inference_config()def get_default_database(self) -> str | None:"""Get the current database name.@@ -691,17 +690,6 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):col_type = engine_type.as_generic()return sql_type_to_data_type(str(col_type))- def _resolve_should_auto_discover(- self,- value: bool | Literal["auto"],- ) -> bool:- if value == "auto":- return self._is_cheap_discovery()- return value-- def _is_cheap_discovery(self) -> bool:- return self.dialect.lower() in CHEAP_DISCOVERY_DATABASES-@staticmethoddef is_cursor_result(result: Any) -> bool:if not DependencyManager.sqlalchemy.has():diff --git a/marimo/_sql/engines/types.py b/marimo/_sql/engines/types.pyindex d62f1e61a8..0d5e3833cb 100644--- a/marimo/_sql/engines/types.py+++ b/marimo/_sql/engines/types.py@@ -14,6 +14,7 @@ from marimo._sql.parse import ()from marimo._sql.utils import (get_configured_sql_output_format,+ is_cheap_dialect,is_query_empty,strip_explain_from_error_message,wrap_query_with_explain,@@ -30,6 +31,19 @@ class InferenceConfig(ABC):auto_discover_columns: bool | Literal["auto"]+def default_inference_config() -> InferenceConfig:+ """Default discovery config shared by general-purpose SQL engines.++ Expensive backends can have a large number of schemas and tables, so we+ gate discovery behind the `"auto"` heuristic.+ """+ return InferenceConfig(+ auto_discover_schemas="auto",+ auto_discover_tables="auto",+ auto_discover_columns=False,+ )++def _validate_sql_output_format(sql_output: SqlOutputType) -> SqlOutputType:if sql_output in ("lazy-polars", "polars"):DependencyManager.polars.require(@@ -121,6 +135,23 @@ class EngineCatalog(BaseEngine[CONN], ABC):) -> DataTable | None:"""Get a single table from the engine."""+ def _resolve_should_auto_discover(+ self, value: bool | Literal["auto"]+ ) -> bool:+ """Resolve a discovery flag, deferring `"auto"` to engine policy."""+ if value == "auto":+ return self._is_cheap_discovery()+ return value++ def _is_cheap_discovery(self) -> bool:+ """Whether discovery is cheap enough to run when a flag is `"auto"`.++ Defaults to a dialect-based heuristic; engines with different cost+ profiles (e.g. always-cheap local catalogs, or expensive remote+ warehouses) should override this.+ """+ return is_cheap_dialect(self.dialect)+class QueryEngine(BaseEngine[CONN], ABC):"""Protocol for SQL engines that can execute queries."""diff --git a/marimo/_sql/utils.py b/marimo/_sql/utils.pyindex d98802a2ae..dadc778357 100644--- a/marimo/_sql/utils.py+++ b/marimo/_sql/utils.py@@ -27,11 +27,28 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()-CHEAP_DISCOVERY_DATABASES = ["duckdb", "sqlite", "mysql", "postgresql"]+CHEAP_DISCOVERY_DATABASES = {+ "duckdb",+ "sqlite",+ "mysql",+ "mariadb",+ "postgresql",+}# DuckDB SQL can return None for DDL, so keep "patch did not apply" distinct._NO_WASM_DUCKDB_RESULT = object()+def is_cheap_dialect(dialect: str) -> bool:+ """Whether schema/table discovery is cheap for the given SQL dialect.++ Used as the default heuristic for resolving `"auto"` discovery flags: local+ or lightweight dialects are cheap to introspect, whereas remote analytical+ backends (e.g. Snowflake, BigQuery) can be slow and should not be scanned+ eagerly.+ """+ return dialect.lower() in CHEAP_DISCOVERY_DATABASES++def get_configured_sql_output_format() -> SqlOutputType:"""Read the configured SQL output format from the runtime context.diff --git a/tests/_sql/test_adbc.py b/tests/_sql/test_adbc.pyindex 1b2bb191ce..7e6c281239 100644--- a/tests/_sql/test_adbc.py+++ b/tests/_sql/test_adbc.py@@ -13,6 +13,7 @@ from marimo._sql.engines.adbc import (_adbc_info_to_dialect,_schema_field_to_data_type,)+from marimo._sql.engines.types import default_inference_configfrom marimo._sql.get_engines import get_engines_from_variablesfrom marimo._types.ids import VariableName@@ -438,6 +439,18 @@ def test_adbc_is_compatible_does_not_create_cursor() -> None:assert conn._cursor.did_close is True # type: ignore[attr-defined]+def test_adbc_engine_uses_default_inference_config() -> None:+ """ADBC shares the default discovery config (see #9775)."""+ conn = FakeAdbcDbApiConnection(+ cursor=FakeAdbcDbApiCursor(description=None),+ objects_pylist=[],+ table_schema=FakeAdbcTableSchema([]),+ )+ engine = AdbcDBAPIEngine(conn)+ assert engine.inference_config == default_inference_config()+ assert engine.inference_config.auto_discover_schemas == "auto"++def test_adbc_catalog_auto_discovery_uses_cheap_dialect_heuristic() -> None:conn = FakeAdbcDbApiConnection(cursor=FakeAdbcDbApiCursor(description=None),diff --git a/tests/_sql/test_clickhouse.py b/tests/_sql/test_clickhouse.pyindex 832350a603..2cce5d74ec 100644--- a/tests/_sql/test_clickhouse.py+++ b/tests/_sql/test_clickhouse.py@@ -99,6 +99,27 @@ def test_clickhouse_server_creation() -> None:assert connection.source == "clickhouse"[email protected](not HAS_PANDAS, reason="pandas not installed")+def test_clickhouse_server_get_databases_auto_skips_tables() -> None:+ """ClickHouse server is not cheap, so `"auto"` must not scan tables."""+ import pandas as pd++ connection = mock.MagicMock()+ connection.query_df.return_value = pd.DataFrame({"name": ["db1", "db2"]})++ engine = ClickhouseServer(connection)+ with mock.patch.object(engine, "get_tables_in_schema") as mock_get_tables:+ databases = engine.get_databases(+ include_schemas="auto",+ include_tables="auto",+ include_table_details="auto",+ )++ mock_get_tables.assert_not_called()+ assert [db.name for db in databases] == ["db1", "db2"]+ assert all(db.schemas[0].tables == [] for db in databases)++@pytest.mark.skipif(not HAS_CHDB and not HAS_POLARS and not HAS_PANDAS,reason="chdb, polars, and pandas not installed",diff --git a/tests/_sql/test_ibis.py b/tests/_sql/test_ibis.pyindex db1fa11569..4b2b44f380 100644--- a/tests/_sql/test_ibis.py+++ b/tests/_sql/test_ibis.py@@ -10,7 +10,11 @@ import pytestfrom marimo._data.models import Database, DataTable, DataTableColumn, Schemafrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._sql.engines.ibis import IbisEngine, IbisToMarimoConversionError-from marimo._sql.engines.types import EngineCatalog, QueryEngine+from marimo._sql.engines.types import (+ EngineCatalog,+ QueryEngine,+ default_inference_config,+)from marimo._sql.sql import sqlfrom marimo._types.ids import VariableName@@ -506,6 +510,16 @@ def test_ibis_engine_get_schemas(ibis_backend: SQLBackend) -> None:assert len(schema.tables) == 0[email protected](not HAS_IBIS, reason="Ibis not installed")+def test_ibis_engine_uses_default_inference_config(+ ibis_backend: SQLBackend,+) -> None:+ """Ibis shares the default discovery config (see #9775)."""+ engine = IbisEngine(ibis_backend, engine_name=VariableName("my_ibis"))+ assert engine.inference_config == default_inference_config()+ assert engine.inference_config.auto_discover_schemas == "auto"++@pytest.mark.skipif(not HAS_IBIS, reason="Ibis not installed")def test_ibis_engine_get_databases(ibis_backend: SQLBackend) -> None:"""Test IbisEngine get_databases method."""diff --git a/tests/_sql/test_sqlalchemy.py b/tests/_sql/test_sqlalchemy.pyindex 188517a9f4..c139354376 100644--- a/tests/_sql/test_sqlalchemy.py+++ b/tests/_sql/test_sqlalchemy.py@@ -11,7 +11,11 @@ import pytestfrom marimo._data.models import Database, DataTable, DataTableColumn, Schemafrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._sql.engines.sqlalchemy import SQLAlchemyEngine, safe_execute-from marimo._sql.engines.types import EngineCatalog, QueryEngine+from marimo._sql.engines.types import (+ EngineCatalog,+ QueryEngine,+ default_inference_config,+)from marimo._sql.sql import sqlfrom marimo._types.ids import VariableName@@ -115,6 +119,17 @@ def test_sqlalchemy_engine_dialect(sqlite_engine: sa.Engine) -> None:assert engine.dialect == "sqlite"[email protected](not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")+def test_sqlalchemy_engine_uses_default_inference_config(+ sqlite_engine: sa.Engine,+) -> None:+ """SQLAlchemy shares the default discovery config (see #9775)."""+ engine = SQLAlchemyEngine(+ sqlite_engine, engine_name=VariableName("test_sqlite")+ )+ assert engine.inference_config == default_inference_config()++@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")def test_sqlalchemy_engine_is_instance(sqlite_engine: sa.Engine) -> None:"""Test SQLAlchemyEngine is an instance of the correct types."""