A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(sql): honor DuckDB discovery settings (#11007)
Shahmir Varqhac1220e6e125a
diff --git a/marimo/_smoke_tests/sql/duckdb_discovery.py b/marimo/_smoke_tests/sql/duckdb_discovery.pynew file mode 100644index 0000000000..15b1a25333--- /dev/null+++ b/marimo/_smoke_tests/sql/duckdb_discovery.py@@ -0,0 +1,348 @@+# Copyright 2026 Marimo. All rights reserved.+# Run from this checkout so the notebook exercises the PR implementation:+# uv run --with duckdb marimo edit marimo/_smoke_tests/sql/duckdb_discovery.py++import marimo++__generated_with = "0.25.0"+app = marimo.App(width="medium")+++def _():+ from itertools import product+ from typing import Any++ import duckdb+ import msgspec++ import marimo as mo+ from marimo._sql.engines.duckdb import DuckDBEngine+ from marimo._types.ids import VariableName++ return Any, DuckDBEngine, VariableName, duckdb, mo, msgspec, product++[email protected](hide_code=True)+def _(duckdb, mo):+ mo.md(f"""+ # DuckDB discovery smoke test++ Verifies issue **#11003** and PR **#11007** against this checkout.+ Uses small in-memory catalogs; no downloads, credentials, or extensions.++ **DuckDB:** `{duckdb.__version__}`++ **marimo loaded from:** `{mo.__file__}`++ The checks below inspect real metadata results and record the SQL sent+ through the engine. They verify query counts, not remote Iceberg latency.+ One check deliberately simulates an unavailable column-metadata query.+ """)+ return+++def _(duckdb):+ def create_smoke_connection() -> duckdb.DuckDBPyConnection:+ connection = duckdb.connect()+ connection.execute("""+ CREATE SCHEMA retail;+ CREATE TABLE retail.orders AS+ SELECT i AS order_id, i * 10 AS amount FROM range(1, 101) t(i);+ CREATE VIEW retail.large_orders AS+ SELECT * FROM retail.orders WHERE amount >= 500;+ CREATE SCHEMA empty_schema;+ CREATE TEMP TABLE scratch AS SELECT 1 AS id;+ ATTACH ':memory:' AS warehouse;+ CREATE SCHEMA warehouse.analytics;+ CREATE TABLE warehouse.analytics.orders AS SELECT 42 AS warehouse_id;+ ATTACH ':memory:' AS "odd""catalog";+ CREATE SCHEMA "odd""catalog"."sales.region";+ CREATE TABLE "odd""catalog"."sales.region"."order""details" (id INTEGER);+ """)+ return connection++ smoke_connection = create_smoke_connection()+ return create_smoke_connection, smoke_connection+++def _(Any, DuckDBEngine, VariableName):+ class ObservedDuckDBEngine(DuckDBEngine):+ """Record this engine's queries without patching notebook-wide state."""++ def __init__(self, connection):+ super().__init__(connection, VariableName("smoke_connection"))+ self.queries: list[dict[str, Any]] = []+ self.fail_columns = False++ def _query_catalog(+ self, query: str, params: list[Any] | None = None+ ) -> list[Any]:+ self.queries.append({"sql": query, "parameters": repr(params)})+ if self.fail_columns and "duckdb_columns()" in query:+ raise RuntimeError("Simulated column catalog failure")+ return super()._query_catalog(query, params)++ return (ObservedDuckDBEngine,)++[email protected](hide_code=True)+def _(mo):+ schema_flag = mo.ui.dropdown(+ {"Disabled": False, "Enabled": True, "Auto": "auto"},+ value="Disabled",+ label="Schemas",+ )+ table_flag = mo.ui.dropdown(+ {"Disabled": False, "Enabled": True, "Auto": "auto"},+ value="Enabled",+ label="Tables",+ )+ column_flag = mo.ui.dropdown(+ {"Disabled": False, "Enabled": True, "Auto": "auto"},+ value="Enabled",+ label="Columns",+ )+ mo.vstack(+ [+ mo.md("""## Explore discovery depth++ These controls pass flags directly to the engine. They do not change+ your marimo settings. Disabled schemas prevent table and column work;+ disabled tables prevent column work. `Auto` retains DuckDB's existing+ cheap-dialect policy and resolves to enabled.+ """),+ mo.hstack([schema_flag, table_flag, column_flag]),+ ]+ )+ return column_flag, schema_flag, table_flag+++def _(+ ObservedDuckDBEngine,+ column_flag,+ schema_flag,+ smoke_connection,+ table_flag,+):+ observed_engine = ObservedDuckDBEngine(smoke_connection)+ discovered_catalogs = observed_engine.get_databases(+ include_schemas=schema_flag.value,+ include_tables=table_flag.value,+ include_table_details=column_flag.value,+ )+ return discovered_catalogs, observed_engine++[email protected](hide_code=True)+def _(discovered_catalogs, mo, msgspec, observed_engine):+ mo.vstack(+ [+ mo.md(f"**Metadata queries: {len(observed_engine.queries)}**"),+ mo.ui.table(observed_engine.queries, selection=None),+ mo.accordion(+ {+ "Returned catalog metadata": mo.json(+ msgspec.to_builtins(discovered_catalogs)+ )+ }+ ),+ ]+ )+ return+++def _(ObservedDuckDBEngine, create_smoke_connection, product):+ check_results = []++ def record_check(name, check) -> None:+ try:+ detail = check()+ except Exception as error:+ check_results.append(+ {"check": name, "status": "FAIL", "detail": repr(error)}+ )+ else:+ check_results.append(+ {"check": name, "status": "PASS", "detail": detail}+ )++ def verify_flags(schemas, tables, columns) -> str:+ with create_smoke_connection() as connection:+ engine = ObservedDuckDBEngine(connection)+ catalogs = engine.get_databases(+ include_schemas=schemas,+ include_tables=tables,+ include_table_details=columns,+ )+ schemas_enabled = schemas is not False+ tables_enabled = schemas_enabled and tables is not False+ columns_enabled = tables_enabled and columns is not False+ queries = [entry["sql"] for entry in engine.queries]+ assert {catalog.name for catalog in catalogs} == {+ "memory",+ "temp",+ "warehouse",+ 'odd"catalog',+ }+ assert sum(+ "duckdb_schemas()" in query for query in queries+ ) == int(schemas_enabled)+ assert sum(+ "duckdb_columns()" in query for query in queries+ ) == int(columns_enabled)+ assert (+ any("information_schema.tables" in query for query in queries)+ == tables_enabled+ )+ for catalog in catalogs:+ assert catalog.schemas_resolved == schemas_enabled+ if not schemas_enabled:+ assert catalog.schemas == []+ for schema in catalog.schemas:+ assert schema.tables_resolved == tables_enabled+ if not tables_enabled:+ assert schema.tables == []+ for table in schema.tables:+ assert bool(table.columns) == columns_enabled+ assert (+ table.num_columns is not None+ ) == columns_enabled+ return f"{len(queries)} queries; schema scans={int(schemas_enabled)}, column scans={int(columns_enabled)}"++ for _schemas, _tables, _columns in product(+ [False, True, "auto"], repeat=3+ ):+ record_check(+ f"Flags: schemas={_schemas}, tables={_tables}, columns={_columns}",+ lambda s=_schemas, t=_tables, c=_columns: verify_flags(s, t, c),+ )++ def verify_deferred_loading() -> str:+ with create_smoke_connection() as connection:+ engine = ObservedDuckDBEngine(connection)+ engine.get_databases(+ include_schemas=False,+ include_tables=False,+ include_table_details=False,+ )+ schemas = engine.get_schemas(+ database="memory",+ include_tables=False,+ include_table_details=False,+ )+ assert "empty_schema" in {schema.name for schema in schemas}+ assert all(not schema.tables_resolved for schema in schemas)+ tables = engine.get_tables_in_schema(+ database="memory", schema="retail", include_table_details=False+ )+ assert [(table.name, table.type) for table in tables] == [+ ("large_orders", "view"),+ ("orders", "table"),+ ]+ assert all(table.columns == [] for table in tables)+ view = engine.get_table_details(+ database_name="memory",+ schema_name="retail",+ table_name="large_orders",+ )+ assert view is not None+ assert view.type == "view"+ assert [column.name for column in view.columns] == [+ "order_id",+ "amount",+ ]+ quoted = engine.get_table_details(+ database_name='odd"catalog',+ schema_name="sales.region",+ table_name='order"details',+ )+ assert quoted is not None+ assert [column.name for column in quoted.columns] == ["id"]+ temporary = engine.get_tables_in_schema(+ database="temp", schema="main", include_table_details=True+ )+ assert [table.name for table in temporary] == ["scratch"]+ assert [column.name for column in temporary[0].columns] == ["id"]+ return "Empty schemas, views, quoted identifiers, and temporary tables load correctly"++ record_check(+ "Deferred expansion and object types", verify_deferred_loading+ )++ def verify_metadata_failure() -> str:+ with create_smoke_connection() as connection:+ engine = ObservedDuckDBEngine(connection)+ engine.fail_columns = True+ tables = engine.get_tables_in_schema(+ database="memory", schema="retail", include_table_details=True+ )+ assert [(table.name, table.type) for table in tables] == [+ ("large_orders", "view"),+ ("orders", "table"),+ ]+ assert all(+ table.columns == [] and table.num_columns is None+ for table in tables+ )+ return "Both objects remain visible after the simulated column failure"++ record_check(+ "Column metadata failure preserves tables", verify_metadata_failure+ )+ return (check_results,)++[email protected](hide_code=True)+def _(check_results, mo):+ _failures = [+ result for result in check_results if result["status"] == "FAIL"+ ]+ mo.vstack(+ [+ mo.md(+ f"## Automated checks: {len(check_results) - len(_failures)}/{len(check_results)} passed"+ ),+ mo.ui.table(check_results, selection=None),+ ]+ )+ return+++def _(check_results):+ assert all(result["status"] == "PASS" for result in check_results), [+ result for result in check_results if result["status"] == "FAIL"+ ]+ return++[email protected](hide_code=True)+def _(mo):+ mo.md("""+ ## Manual datasource-panel check++ 1. Open the **Data Sources** panel and find `smoke_connection`.+ 2. Expand `memory` → `retail`. `large_orders` should be a view and+ `orders` a table. Expand each to load `order_id` and `amount`.+ 3. Expand `temp` → `main` → `scratch`, and the quoted catalog and table.+ 4. Disable schema, table, and column auto-discovery in datasource settings,+ refresh the connection, and repeat the expansions. Restore your settings+ afterward. The controls above affect only the recorded engine calls.++ **Expected warning:** the failure check intentionally logs+ `Simulated column catalog failure`. Its result should still be **PASS**.++ This notebook cannot establish real Iceberg network performance. A remote+ REST catalog with credentials is needed for that measurement; a public CSV+ would exercise data reads rather than catalog discovery.+ """)+ return+++if __name__ == "__main__":+ app.run()diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.pyindex 84f41f2c84..66150a9a93 100644--- a/marimo/_sql/engines/duckdb.py+++ b/marimo/_sql/engines/duckdb.py@@ -5,14 +5,21 @@ from contextlib import contextmanager, nullcontextfrom typing import TYPE_CHECKING, Any, Literal, Optional, castfrom marimo import _loggers-from marimo._data.get_datasets import get_databases_from_duckdb-from marimo._data.models import Database, DataTable, Schema+from marimo._data.get_datasets import _db_type_to_data_type, get_table_columns+from marimo._data.models import (+ Database,+ DataTable,+ DataTableColumn,+ DataTableType,+ Schema,+)from marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime.context.types import (ContextNotInitializedError,get_context,)from marimo._sql.engines.types import InferenceConfig, SQLConnection+from marimo._sql.sql_quoting import quote_qualified_namefrom marimo._sql.utils import convert_to_output, wrapped_sqlfrom marimo._types.ids import VariableName@@ -120,7 +127,6 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):@propertydef inference_config(self) -> InferenceConfig:- # At the moment this isn't being used for duckdbreturn InferenceConfig(auto_discover_schemas=True,auto_discover_tables="auto",@@ -166,18 +172,72 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):include_tables: bool | Literal["auto"],include_table_details: bool | Literal["auto"],) -> list[Database]:- """Fetch all databases from the engine. At the moment, will fetch everything."""- _, _, _ = include_schemas, include_tables, include_table_details+ """List databases, discovering only the requested metadata depth."""+ schemas_resolved = self._resolve_should_auto_discover(include_schemas)+ tables_resolved = self._resolve_should_auto_discover(include_tables)+ details_resolved = self._resolve_should_auto_discover(+ include_table_details+ )+ # Keep the temporary catalog accessible without scanning its tables.+ rows = self._query_catalog(+ "SELECT database_name FROM duckdb_databases() "+ "WHERE NOT internal OR database_name = 'temp' "+ "ORDER BY database_name"+ )+ databases = {+ name: Database(+ name=name,+ dialect=self.dialect,+ engine=self._engine_name,+ schemas_resolved=schemas_resolved,+ schemas=[],+ )+ for (name,) in rows+ }+ if schemas_resolved and databases:+ # duckdb_schemas() scans every catalog before applying filters.+ # Query it once so remote namespaces are not fetched per database.+ schema_rows = self._query_catalog(+ "SELECT database_name, schema_name FROM duckdb_schemas() "+ "WHERE schema_name NOT IN ('information_schema', 'pg_catalog') "+ "ORDER BY database_name, schema_name"+ )+ for database_name, schema_name in schema_rows:+ if database_name in databases:+ databases[database_name].schemas.append(+ self._make_schema(+ schema_name,+ database_name,+ include_tables=tables_resolved,+ include_table_details=False,+ )+ )+ if details_resolved:+ self._load_table_details(+ {+ (database.name, schema.name, table.name): table+ for database in databases.values()+ for schema in database.schemas+ for table in schema.tables+ }+ )+ return list(databases.values())++ def _query_catalog(+ self, query: str, params: list[Any] | None = None+ ) -> list[Any]:import duckdbconnection = cast(duckdb.DuckDBPyConnection, self._connection or duckdb)- with self._install_connection(connection):- return get_databases_from_duckdb(connection, self._engine_name)+ try:+ with self._install_connection(connection):+ return connection.execute(query, params).fetchall()+ except duckdb.ConnectionException:+ LOGGER.debug("Skipping closed DuckDB connection")+ return []- # TODO: The following methods are currently not implemented.- # We should consider implementing these in the future for better performance when users don't want to fetch everything.def get_schemas(self,*,@@ -186,14 +246,46 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):include_table_details: bool,schema_path: list[str] | None = None,) -> list[Schema]:- """Get all schemas and optionally their tables. Keys are schema names."""- _, _, _, _ = (- database,- include_tables,- include_table_details,- schema_path,+ """List schemas without enumerating tables unless requested."""+ if schema_path:+ return []+ if database is None:+ database = self.get_default_database()+ if database is None:+ return []+ rows = self._query_catalog(+ "SELECT schema_name FROM duckdb_schemas() "+ "WHERE database_name = ? "+ "AND schema_name NOT IN ('information_schema', 'pg_catalog') "+ "ORDER BY schema_name",+ [database],)- return []+ return [+ self._make_schema(+ name,+ database,+ include_tables=include_tables,+ include_table_details=include_table_details,+ )+ for (name,) in rows+ ]++ def _make_schema(+ self,+ name: str,+ database: str,+ *,+ include_tables: bool,+ include_table_details: bool,+ ) -> Schema:+ tables = []+ if include_tables:+ tables = self.get_tables_in_schema(+ schema=name,+ database=database,+ include_table_details=include_table_details,+ )+ return Schema(name=name, tables=tables, tables_resolved=include_tables)def get_tables_in_schema(self,@@ -203,9 +295,89 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):include_table_details: bool,schema_path: list[str] | None = None,) -> list[DataTable]:- """Return all tables in a schema. This is currently implemented in get_databases_from_duckdb."""- _, _, _, _ = database, schema, include_table_details, schema_path- return []+ """List table and view names without eagerly loading their columns."""+ del schema_path+ rows = self._query_catalog(+ "SELECT table_name, table_type FROM information_schema.tables "+ "WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",+ [database, schema],+ )+ tables = {+ (database, schema, name): self._make_table(+ name,+ database,+ table_type="view" if kind == "VIEW" else "table",+ )+ for name, kind in rows+ }+ if include_table_details:+ self._load_table_details(tables, database=database, schema=schema)+ return list(tables.values())++ def _load_table_details(+ self,+ tables: dict[tuple[str, str, str], DataTable],+ *,+ database: str | None = None,+ schema: str | None = None,+ ) -> None:+ if not tables:+ return+ query = (+ "SELECT database_name, schema_name, table_name, column_name, data_type "+ "FROM duckdb_columns() WHERE NOT internal"+ )+ params: list[str] = []+ if database is not None:+ query += " AND database_name = ?"+ params.append(database)+ if schema is not None:+ query += " AND schema_name = ?"+ params.append(schema)+ query += (+ " ORDER BY database_name, schema_name, table_name, column_index"+ )+ try:+ rows = self._query_catalog(query, params)+ except Exception:+ # A metadata failure must not hide tables already enumerated.+ LOGGER.warning("Failed to get DuckDB columns", exc_info=True)+ return+ for db_name, schema_name, table_name, column_name, dtype in rows:+ table = tables.get((db_name, schema_name, table_name))+ if table is not None:+ table.columns.append(+ DataTableColumn(+ name=column_name,+ type=_db_type_to_data_type(dtype),+ external_type=dtype,+ sample_values=[],+ )+ )+ for (_, schema_name, _), table in tables.items():+ # Some Iceberg catalogs expose a placeholder instead of columns.+ if len(table.columns) == 1 and table.columns[0].name == "__":+ table.columns = []+ self._describe_table(table, schema_name)+ if table.columns:+ table.num_columns = len(table.columns)++ def _make_table(+ self, name: str, database: str, *, table_type: DataTableType = "table"+ ) -> DataTable:+ return DataTable(+ source_type="duckdb"+ if self._engine_name is None+ else "connection",+ source=database,+ name=name,+ type=table_type,+ num_rows=None,+ num_columns=None,+ variable_name=None,+ columns=[],+ engine=self._engine_name,+ )def get_table_details(self,@@ -215,6 +387,31 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):database_name: str,schema_path: list[str] | None = None,) -> DataTable | None:- """Get a single table from the engine. This is currently implemented in get_databases_from_duckdb."""- _, _, _, _ = table_name, schema_name, database_name, schema_path- return None+ """Describe only the requested table, including remote catalog tables."""+ del schema_path+ rows = self._query_catalog(+ "SELECT table_type FROM information_schema.tables "+ "WHERE table_catalog = ? AND table_schema = ? AND table_name = ?",+ [database_name, schema_name, table_name],+ )+ if not rows:+ return None+ table = self._make_table(+ table_name,+ database_name,+ table_type="view" if rows[0][0] == "VIEW" else "table",+ )+ self._describe_table(table, schema_name)+ return table if table.columns else None++ def _describe_table(self, table: DataTable, schema: str) -> None:+ import duckdb++ connection = cast(+ duckdb.DuckDBPyConnection, self._connection or duckdb+ )+ qualified_name = quote_qualified_name(table.source, schema, table.name)+ with self._install_connection(connection):+ table.columns = get_table_columns(connection, qualified_name)+ if table.columns:+ table.num_columns = len(table.columns)diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.pyindex 720d0e057c..82e74991a2 100644--- a/tests/_runtime/test_runtime_datasets.py+++ b/tests/_runtime/test_runtime_datasets.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport pytest+from marimo._data.models import DataTablefrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.notification import (DataSourceConnectionsNotification,@@ -24,7 +25,7 @@ from marimo._runtime.commands import ()from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINEfrom marimo._sql.parse import SqlCatalogCheckResult, SqlParseResult-from marimo._types.ids import CellId_t, RequestId+from marimo._types.ids import CellId_t, RequestId, VariableNamefrom tests.conftest import MockedKernelHAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()@@ -360,13 +361,21 @@ class TestPreviewSQLTableList:k = mocked_kernel.kstream = mocked_kernel.stream- await k.run(connection_requests)+ await k.run(+ [+ *connection_requests,+ ExecuteCellCommand(+ cell_id=CellId_t("4"),+ code=f"{DUCKDB_CONN}.execute('CREATE TABLE example (id INTEGER)')",+ ),+ ]+ )preview_sql_table_list_request = ListSQLTablesCommand(request_id=RequestId("0"),engine=DUCKDB_CONN,- database="test",- schema="test",+ database="memory",+ schema="main",)await k.handle_message(preview_sql_table_list_request)@@ -378,10 +387,21 @@ class TestPreviewSQLTableList:assert preview_sql_table_list_results == [SQLTableListPreviewNotification(request_id=RequestId("0"),- tables=[],+ tables=[+ DataTable(+ name="example",+ source="memory",+ source_type="connection",+ engine=VariableName(DUCKDB_CONN),+ num_rows=None,+ num_columns=None,+ columns=[],+ variable_name=None,+ )+ ],error=None,metadata=SQLMetadata(- connection=DUCKDB_CONN, database="test", schema="test"+ connection=DUCKDB_CONN, database="memory", schema="main"),)]diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.pyindex f05a8a888a..208971c1f3 100644--- a/tests/_sql/test_duckdb.py+++ b/tests/_sql/test_duckdb.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport sysfrom copy import deepcopy-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Literalfrom unittest import mockimport pytest@@ -13,7 +13,9 @@ from marimo._data.models import Database, DataTable, DataTableColumn, Schemafrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._sql.engines.duckdb import DuckDBEnginefrom marimo._sql.engines.types import EngineCatalog, QueryEngine+from marimo._sql.get_engines import engine_to_data_source_connectionfrom marimo._sql.sql import sql+from marimo._sql.sql_quoting import quote_qualified_namefrom marimo._types.ids import VariableNameHAS_DUCKDB = DependencyManager.duckdb.has()@@ -140,7 +142,15 @@ def test_duckdb_engine_get_databases(include_schemas=True, include_tables=True, include_table_details=True)- assert databases == expected_databases_with_conn+ assert databases == [+ *expected_databases_with_conn,+ Database(+ name="temp",+ dialect="duckdb",+ engine=VariableName("test_duckdb"),+ schemas=[Schema(name="main", tables=[])],+ ),+ ]@pytest.mark.skipif(not HAS_DUCKDB, reason="DuckDB not installed")@@ -153,7 +163,10 @@ def test_duckdb_engine_get_databases_no_conn() -> None:include_tables=False,)assert initial_databases == [- Database(name="memory", dialect="duckdb", schemas=[])+ Database(+ name=name, dialect="duckdb", schemas=[], schemas_resolved=False+ )+ for name in ("memory", "temp")]assert engine.get_default_database() == "memory"assert engine.get_default_schema() == "main"@@ -179,7 +192,14 @@ def test_duckdb_engine_get_databases_no_conn() -> None:expected_databases[0].schemas[0].tables[0].source_type = "duckdb"expected_databases[0].schemas[0].tables[0].source = "memory"- assert databases == expected_databases+ assert databases == [+ *expected_databases,+ Database(+ name="temp",+ dialect="duckdb",+ schemas=[Schema(name="main", tables=[])],+ ),+ ]engine.execute("DROP TABLE test")@@ -439,3 +459,381 @@ def test_duckdb_engine_lazy_polars_falls_back_on_older_duckdb(# Only the first call (lazy=True, raises) reaches `pl`; the fallback uses# `to_polars()` (Arrow PyCapsule) and never touches `relation.pl()`.assert pl_calls == [{"batch_size": 100_000, "lazy": True}]++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")[email protected]("include_schemas", [False, True, "auto"])[email protected]("include_tables", [False, True, "auto"])[email protected]("include_details", [False, True, "auto"])+def test_duckdb_discovery_depth(+ include_schemas: bool | Literal["auto"],+ include_tables: bool | Literal["auto"],+ include_details: bool | Literal["auto"],+) -> None:+ import duckdb++ with duckdb.connect() as connection:+ connection.execute("CREATE TABLE test (id INTEGER, name VARCHAR)")+ engine = DuckDBEngine(connection, VariableName("test_duckdb"))+ with (+ mock.patch.object(+ engine, "_query_catalog", wraps=engine._query_catalog+ ) as query,+ mock.patch.object(+ engine, "get_table_details", wraps=engine.get_table_details+ ) as details,+ ):+ databases = engine.get_databases(+ include_schemas=include_schemas,+ include_tables=include_tables,+ include_table_details=include_details,+ )++ schemas_enabled = include_schemas is not False+ tables_enabled = schemas_enabled and include_tables is not False+ details_enabled = tables_enabled and include_details is not False+ expected = deepcopy(expected_databases_with_conn[0])+ if not schemas_enabled:+ expected.schemas = []+ expected.schemas_resolved = False+ elif not tables_enabled:+ expected.schemas[0].tables = []+ expected.schemas[0].tables_resolved = False+ elif not details_enabled:+ expected.schemas[0].tables[0].columns = []+ expected.schemas[0].tables[0].num_columns = None+ assert databases[0] == expected+ queries = [call.args[0] for call in query.call_args_list]+ assert sum("duckdb_schemas()" in sql for sql in queries) == int(+ schemas_enabled+ )+ assert (+ any("information_schema.tables" in sql for sql in queries)+ == tables_enabled+ )+ assert sum("duckdb_columns()" in sql for sql in queries) == int(+ details_enabled+ )+ assert details.call_count == 0+ assert not any("SHOW ALL TABLES" in sql for sql in queries)++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+def test_duckdb_deferred_discovery() -> None:+ import duckdb++ database, schema, table_name = 'db".quoted', "schema's.name", 'table".name'+ qualified_table = quote_qualified_name(database, schema, table_name)+ with duckdb.connect() as connection:+ connection.execute(+ f"ATTACH ':memory:' AS {quote_qualified_name(database)}"+ )+ connection.execute(+ f"CREATE SCHEMA {quote_qualified_name(database, schema)}"+ )+ connection.execute(+ f"CREATE TABLE {qualified_table} (id INTEGER, name VARCHAR)"+ )+ connection.execute(+ f"CREATE VIEW {quote_qualified_name(database, schema, 'a_view')} AS SELECT * FROM {qualified_table}"+ )+ connection.execute("CREATE TEMP TABLE local_temp (id INTEGER)")+ engine = DuckDBEngine(connection, VariableName("test_duckdb"))+ databases = engine.get_databases(+ include_schemas=False,+ include_tables=False,+ include_table_details=False,+ )+ assert {db.name for db in databases} == {database, "memory", "temp"}+ assert all(+ not db.schemas_resolved and not db.schemas for db in databases+ )++ schemas = engine.get_schemas(+ database=database,+ include_tables=False,+ include_table_details=False,+ )+ assert schemas == [+ Schema(name="main", tables=[], tables_resolved=False),+ Schema(name=schema, tables=[], tables_resolved=False),+ ]+ tables = engine.get_tables_in_schema(+ database=database, schema=schema, include_table_details=False+ )+ assert [(table.name, table.type) for table in tables] == [+ ("a_view", "view"),+ (table_name, "table"),+ ]+ assert all(+ table.columns == [] and table.num_columns is None+ for table in tables+ )+ expected = deepcopy(+ expected_databases_with_conn[0].schemas[0].tables[0]+ )+ expected.source = database+ for name in (table_name, "a_view"):+ expected.name = name+ expected.type = "view" if name == "a_view" else "table"+ assert (+ engine.get_table_details(+ database_name=database, schema_name=schema, table_name=name+ )+ == expected+ )+ assert (+ engine.get_table_details(+ database_name=database,+ schema_name=schema,+ table_name="missing",+ )+ is None+ )+ assert (+ engine.get_schemas(+ database=database,+ include_tables=False,+ include_table_details=False,+ schema_path=[schema],+ )+ == []+ )+ assert [+ table.name+ for table in engine.get_tables_in_schema(+ database="temp", schema="main", include_table_details=False+ )+ ] == ["local_temp"]+ assert engine.get_schemas(+ database=None, include_tables=False, include_table_details=False+ ) == [Schema(name="main", tables=[], tables_resolved=False)]++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")[email protected]("disable_discovery", [False, True])+def test_duckdb_discovery_config(disable_discovery: bool) -> None:+ import duckdb++ with duckdb.connect() as connection:+ connection.execute("CREATE TABLE test (id INTEGER)")+ name = VariableName("connection")+ engine = DuckDBEngine(connection, name)+ config = {}+ if disable_discovery:+ config = {+ "auto_discover_schemas": False,+ "auto_discover_tables": False,+ "auto_discover_columns": False,+ }+ with mock.patch(+ "marimo._sql.get_engines.get_datasources_config",+ return_value=config,+ ):+ source = engine_to_data_source_connection(name, engine)+ if disable_discovery:+ assert source.databases == [+ Database(+ name=db,+ dialect="duckdb",+ engine=name,+ schemas=[],+ schemas_resolved=False,+ )+ for db in ("memory", "temp")+ ]+ else:+ # The default discovers table names but defers columns.+ assert source.databases[0].schemas[0].tables == [+ DataTable(+ name="test",+ source="memory",+ source_type="connection",+ engine=name,+ num_rows=None,+ num_columns=None,+ columns=[],+ variable_name=None,+ )+ ]++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+def test_duckdb_discovery_closed_connection() -> None:+ import duckdb++ connection = duckdb.connect()+ connection.close()+ engine = DuckDBEngine(connection)+ assert (+ engine.get_databases(+ include_schemas=True,+ include_tables=True,+ include_table_details=True,+ )+ == []+ )++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+def test_duckdb_discovers_schemas_once_for_all_databases() -> None:+ import duckdb++ with duckdb.connect() as connection:+ for name in ("first", "second"):+ connection.execute(f"ATTACH ':memory:' AS {name}")+ connection.execute(f"CREATE SCHEMA {name}.extra")+ engine = DuckDBEngine(connection)+ with mock.patch.object(+ engine, "_query_catalog", wraps=engine._query_catalog+ ) as query:+ databases = engine.get_databases(+ include_schemas=True,+ include_tables=False,+ include_table_details=False,+ )+ assert databases == [+ Database(+ name=name,+ dialect="duckdb",+ schemas=[+ Schema(name=schema, tables=[], tables_resolved=False)+ for schema in schemas+ ],+ )+ for name, schemas in [+ ("first", ["extra", "main"]),+ ("memory", ["main"]),+ ("second", ["extra", "main"]),+ ("temp", ["main"]),+ ]+ ]+ assert len(query.call_args_list) == 2+ assert (+ sum(+ "duckdb_schemas()" in call.args[0]+ for call in query.call_args_list+ )+ == 1+ )++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")[email protected]("all_databases", [False, True])+def test_duckdb_batches_column_discovery(all_databases: bool) -> None:+ import duckdb++ with duckdb.connect() as connection:+ connection.execute("ATTACH ':memory:' AS attached")+ for database in ("memory", "attached"):+ connection.execute(f"CREATE SCHEMA {database}.extra")+ for schema in ("main", "extra"):+ connection.execute(+ f"CREATE TABLE {database}.{schema}.example (z INTEGER, a VARCHAR)"+ )+ connection.execute(+ f"CREATE VIEW {database}.{schema}.example_view AS SELECT * FROM {database}.{schema}.example"+ )+ engine = DuckDBEngine(connection)+ with (+ mock.patch.object(+ engine, "_query_catalog", wraps=engine._query_catalog+ ) as query,+ mock.patch.object(+ engine, "_describe_table", wraps=engine._describe_table+ ) as describe,+ ):+ if all_databases:+ databases = engine.get_databases(+ include_schemas=True,+ include_tables=True,+ include_table_details=True,+ )+ tables = [+ table+ for database in databases+ for schema in database.schemas+ for table in schema.tables+ ]+ else:+ tables = engine.get_tables_in_schema(+ database="attached",+ schema="main",+ include_table_details=True,+ )+ assert len(tables) == (8 if all_databases else 2)+ for table in tables:+ assert table.columns == [+ DataTableColumn(+ name="z",+ type="integer",+ external_type="INTEGER",+ sample_values=[],+ ),+ DataTableColumn(+ name="a",+ type="string",+ external_type="VARCHAR",+ sample_values=[],+ ),+ ]+ assert table.num_columns == 2+ assert table.type == (+ "view" if table.name == "example_view" else "table"+ )+ assert (+ sum(+ "duckdb_columns()" in call.args[0]+ for call in query.call_args_list+ )+ == 1+ )+ describe.assert_not_called()++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")[email protected]("failure", ["query", "missing", "placeholder"])+def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:+ import duckdb++ with duckdb.connect() as connection:+ connection.execute("CREATE TABLE example (id INTEGER)")+ engine = DuckDBEngine(connection)+ query = engine._query_catalog++ def failing_columns(+ sql: str, params: list[Any] | None = None+ ) -> list[Any]:+ if "duckdb_columns()" in sql:+ if failure == "query":+ raise duckdb.NotImplementedException(+ "Column metadata unavailable"+ )+ if failure == "placeholder":+ return [("memory", "main", "example", "__", "INTEGER")]+ return []+ return query(sql, params)++ with (+ mock.patch.object(+ engine, "_query_catalog", side_effect=failing_columns+ ),+ mock.patch(+ "marimo._sql.engines.duckdb.get_table_columns", return_value=[]+ ) as describe,+ ):+ tables = engine.get_tables_in_schema(+ database="memory", schema="main", include_table_details=True+ )+ assert tables == [+ DataTable(+ name="example",+ source="memory",+ source_type="duckdb",+ num_rows=None,+ num_columns=None,+ variable_name=None,+ columns=[],+ )+ ]+ assert describe.call_count == int(failure == "placeholder")diff --git a/tests/_sql/test_get_engines.py b/tests/_sql/test_get_engines.pyindex b202f0f6bb..9f91469adf 100644--- a/tests/_sql/test_get_engines.py+++ b/tests/_sql/test_get_engines.py@@ -8,7 +8,6 @@ import pytestfrom marimo._data.models import (Database,DataSourceConnection,- DataTableColumn,Schema,)from marimo._dependencies.dependencies import DependencyManager@@ -52,7 +51,12 @@ def test_engine_to_data_source_connection() -> None:assert connection.default_database == "memory"assert connection.default_schema == "main"assert connection.databases == [- Database(name="memory", dialect="duckdb", schemas=[])+ Database(+ name=name,+ dialect="duckdb",+ schemas=[Schema(name="main", tables=[])],+ )+ for name in ("memory", "temp")]# Test with ClickhouseEmbedded engine@@ -357,7 +361,10 @@ def test_get_engines_duckdb_databases() -> None:VariableName("my_duckdb"), duckdb_engine)- assert len(connection.databases) == 1+ assert [database.name for database in connection.databases] == [+ "memory",+ "temp",+ ]database = connection.databases[0]assert database.name == "memory"assert len(database.schemas) == 1@@ -366,14 +373,8 @@ def test_get_engines_duckdb_databases() -> None:assert len(schema.tables) == 1table = schema.tables[0]assert table.name == "test_table"- assert table.columns == [- DataTableColumn(- name="id",- type="integer",- external_type="INTEGER",- sample_values=[],- )- ]+ assert table.columns == []+ assert table.num_columns is Nonewith patch("marimo._sql.sql.get_configured_sql_output_format",return_value="native",