Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(sql): honor DuckDB discovery settings (#11007)

Shahmir Varqhac1220e6e125a
diff --git a/marimo/_smoke_tests/sql/duckdb_discovery.py b/marimo/_smoke_tests/sql/duckdb_discovery.py
new file mode 100644
index 0000000000..15b1a25333
--- /dev/null
+++ b/marimo/_smoke_tests/sql/duckdb_discovery.py
@@ -0,0 +1,348 @@
+# Copyright 2026 Marimo. All rights reserved.
+# Run from this checkout so the notebook exercises the PR implementation:
+# uv run --with duckdb marimo edit marimo/_smoke_tests/sql/duckdb_discovery.py
+
+import marimo
+
+__generated_with = "0.25.0"
+app = marimo.App(width="medium")
+
+
+def _():
+ from itertools import product
+ from typing import Any
+
+ import duckdb
+ import msgspec
+
+ import marimo as mo
+ from marimo._sql.engines.duckdb import DuckDBEngine
+ from marimo._types.ids import VariableName
+
+ return Any, DuckDBEngine, VariableName, duckdb, mo, msgspec, product
+
+
[email protected](hide_code=True)
+def _(duckdb, mo):
+ mo.md(f"""
+ # DuckDB discovery smoke test
+
+ Verifies issue **#11003** and PR **#11007** against this checkout.
+ Uses small in-memory catalogs; no downloads, credentials, or extensions.
+
+ **DuckDB:** `{duckdb.__version__}`
+
+ **marimo loaded from:** `{mo.__file__}`
+
+ The checks below inspect real metadata results and record the SQL sent
+ through the engine. They verify query counts, not remote Iceberg latency.
+ One check deliberately simulates an unavailable column-metadata query.
+ """)
+ return
+
+
+def _(duckdb):
+ def create_smoke_connection() -> duckdb.DuckDBPyConnection:
+ connection = duckdb.connect()
+ connection.execute("""
+ CREATE SCHEMA retail;
+ CREATE TABLE retail.orders AS
+ SELECT i AS order_id, i * 10 AS amount FROM range(1, 101) t(i);
+ CREATE VIEW retail.large_orders AS
+ SELECT * FROM retail.orders WHERE amount >= 500;
+ CREATE SCHEMA empty_schema;
+ CREATE TEMP TABLE scratch AS SELECT 1 AS id;
+ ATTACH ':memory:' AS warehouse;
+ CREATE SCHEMA warehouse.analytics;
+ CREATE TABLE warehouse.analytics.orders AS SELECT 42 AS warehouse_id;
+ ATTACH ':memory:' AS "odd""catalog";
+ CREATE SCHEMA "odd""catalog"."sales.region";
+ CREATE TABLE "odd""catalog"."sales.region"."order""details" (id INTEGER);
+ """)
+ return connection
+
+ smoke_connection = create_smoke_connection()
+ return create_smoke_connection, smoke_connection
+
+
+def _(Any, DuckDBEngine, VariableName):
+ class ObservedDuckDBEngine(DuckDBEngine):
+ """Record this engine's queries without patching notebook-wide state."""
+
+ def __init__(self, connection):
+ super().__init__(connection, VariableName("smoke_connection"))
+ self.queries: list[dict[str, Any]] = []
+ self.fail_columns = False
+
+ def _query_catalog(
+ self, query: str, params: list[Any] | None = None
+ ) -> list[Any]:
+ self.queries.append({"sql": query, "parameters": repr(params)})
+ if self.fail_columns and "duckdb_columns()" in query:
+ raise RuntimeError("Simulated column catalog failure")
+ return super()._query_catalog(query, params)
+
+ return (ObservedDuckDBEngine,)
+
+
[email protected](hide_code=True)
+def _(mo):
+ schema_flag = mo.ui.dropdown(
+ {"Disabled": False, "Enabled": True, "Auto": "auto"},
+ value="Disabled",
+ label="Schemas",
+ )
+ table_flag = mo.ui.dropdown(
+ {"Disabled": False, "Enabled": True, "Auto": "auto"},
+ value="Enabled",
+ label="Tables",
+ )
+ column_flag = mo.ui.dropdown(
+ {"Disabled": False, "Enabled": True, "Auto": "auto"},
+ value="Enabled",
+ label="Columns",
+ )
+ mo.vstack(
+ [
+ mo.md("""## Explore discovery depth
+
+ These controls pass flags directly to the engine. They do not change
+ your marimo settings. Disabled schemas prevent table and column work;
+ disabled tables prevent column work. `Auto` retains DuckDB's existing
+ cheap-dialect policy and resolves to enabled.
+ """),
+ mo.hstack([schema_flag, table_flag, column_flag]),
+ ]
+ )
+ return column_flag, schema_flag, table_flag
+
+
+def _(
+ ObservedDuckDBEngine,
+ column_flag,
+ schema_flag,
+ smoke_connection,
+ table_flag,
+):
+ observed_engine = ObservedDuckDBEngine(smoke_connection)
+ discovered_catalogs = observed_engine.get_databases(
+ include_schemas=schema_flag.value,
+ include_tables=table_flag.value,
+ include_table_details=column_flag.value,
+ )
+ return discovered_catalogs, observed_engine
+
+
[email protected](hide_code=True)
+def _(discovered_catalogs, mo, msgspec, observed_engine):
+ mo.vstack(
+ [
+ mo.md(f"**Metadata queries: {len(observed_engine.queries)}**"),
+ mo.ui.table(observed_engine.queries, selection=None),
+ mo.accordion(
+ {
+ "Returned catalog metadata": mo.json(
+ msgspec.to_builtins(discovered_catalogs)
+ )
+ }
+ ),
+ ]
+ )
+ return
+
+
+def _(ObservedDuckDBEngine, create_smoke_connection, product):
+ check_results = []
+
+ def record_check(name, check) -> None:
+ try:
+ detail = check()
+ except Exception as error:
+ check_results.append(
+ {"check": name, "status": "FAIL", "detail": repr(error)}
+ )
+ else:
+ check_results.append(
+ {"check": name, "status": "PASS", "detail": detail}
+ )
+
+ def verify_flags(schemas, tables, columns) -> str:
+ with create_smoke_connection() as connection:
+ engine = ObservedDuckDBEngine(connection)
+ catalogs = engine.get_databases(
+ include_schemas=schemas,
+ include_tables=tables,
+ include_table_details=columns,
+ )
+ schemas_enabled = schemas is not False
+ tables_enabled = schemas_enabled and tables is not False
+ columns_enabled = tables_enabled and columns is not False
+ queries = [entry["sql"] for entry in engine.queries]
+ assert {catalog.name for catalog in catalogs} == {
+ "memory",
+ "temp",
+ "warehouse",
+ 'odd"catalog',
+ }
+ assert sum(
+ "duckdb_schemas()" in query for query in queries
+ ) == int(schemas_enabled)
+ assert sum(
+ "duckdb_columns()" in query for query in queries
+ ) == int(columns_enabled)
+ assert (
+ any("information_schema.tables" in query for query in queries)
+ == tables_enabled
+ )
+ for catalog in catalogs:
+ assert catalog.schemas_resolved == schemas_enabled
+ if not schemas_enabled:
+ assert catalog.schemas == []
+ for schema in catalog.schemas:
+ assert schema.tables_resolved == tables_enabled
+ if not tables_enabled:
+ assert schema.tables == []
+ for table in schema.tables:
+ assert bool(table.columns) == columns_enabled
+ assert (
+ table.num_columns is not None
+ ) == columns_enabled
+ return f"{len(queries)} queries; schema scans={int(schemas_enabled)}, column scans={int(columns_enabled)}"
+
+ for _schemas, _tables, _columns in product(
+ [False, True, "auto"], repeat=3
+ ):
+ record_check(
+ f"Flags: schemas={_schemas}, tables={_tables}, columns={_columns}",
+ lambda s=_schemas, t=_tables, c=_columns: verify_flags(s, t, c),
+ )
+
+ def verify_deferred_loading() -> str:
+ with create_smoke_connection() as connection:
+ engine = ObservedDuckDBEngine(connection)
+ engine.get_databases(
+ include_schemas=False,
+ include_tables=False,
+ include_table_details=False,
+ )
+ schemas = engine.get_schemas(
+ database="memory",
+ include_tables=False,
+ include_table_details=False,
+ )
+ assert "empty_schema" in {schema.name for schema in schemas}
+ assert all(not schema.tables_resolved for schema in schemas)
+ tables = engine.get_tables_in_schema(
+ database="memory", schema="retail", include_table_details=False
+ )
+ assert [(table.name, table.type) for table in tables] == [
+ ("large_orders", "view"),
+ ("orders", "table"),
+ ]
+ assert all(table.columns == [] for table in tables)
+ view = engine.get_table_details(
+ database_name="memory",
+ schema_name="retail",
+ table_name="large_orders",
+ )
+ assert view is not None
+ assert view.type == "view"
+ assert [column.name for column in view.columns] == [
+ "order_id",
+ "amount",
+ ]
+ quoted = engine.get_table_details(
+ database_name='odd"catalog',
+ schema_name="sales.region",
+ table_name='order"details',
+ )
+ assert quoted is not None
+ assert [column.name for column in quoted.columns] == ["id"]
+ temporary = engine.get_tables_in_schema(
+ database="temp", schema="main", include_table_details=True
+ )
+ assert [table.name for table in temporary] == ["scratch"]
+ assert [column.name for column in temporary[0].columns] == ["id"]
+ return "Empty schemas, views, quoted identifiers, and temporary tables load correctly"
+
+ record_check(
+ "Deferred expansion and object types", verify_deferred_loading
+ )
+
+ def verify_metadata_failure() -> str:
+ with create_smoke_connection() as connection:
+ engine = ObservedDuckDBEngine(connection)
+ engine.fail_columns = True
+ tables = engine.get_tables_in_schema(
+ database="memory", schema="retail", include_table_details=True
+ )
+ assert [(table.name, table.type) for table in tables] == [
+ ("large_orders", "view"),
+ ("orders", "table"),
+ ]
+ assert all(
+ table.columns == [] and table.num_columns is None
+ for table in tables
+ )
+ return "Both objects remain visible after the simulated column failure"
+
+ record_check(
+ "Column metadata failure preserves tables", verify_metadata_failure
+ )
+ return (check_results,)
+
+
[email protected](hide_code=True)
+def _(check_results, mo):
+ _failures = [
+ result for result in check_results if result["status"] == "FAIL"
+ ]
+ mo.vstack(
+ [
+ mo.md(
+ f"## Automated checks: {len(check_results) - len(_failures)}/{len(check_results)} passed"
+ ),
+ mo.ui.table(check_results, selection=None),
+ ]
+ )
+ return
+
+
+def _(check_results):
+ assert all(result["status"] == "PASS" for result in check_results), [
+ result for result in check_results if result["status"] == "FAIL"
+ ]
+ return
+
+
[email protected](hide_code=True)
+def _(mo):
+ mo.md("""
+ ## Manual datasource-panel check
+
+ 1. Open the **Data Sources** panel and find `smoke_connection`.
+ 2. Expand `memory` → `retail`. `large_orders` should be a view and
+ `orders` a table. Expand each to load `order_id` and `amount`.
+ 3. Expand `temp` → `main` → `scratch`, and the quoted catalog and table.
+ 4. Disable schema, table, and column auto-discovery in datasource settings,
+ refresh the connection, and repeat the expansions. Restore your settings
+ afterward. The controls above affect only the recorded engine calls.
+
+ **Expected warning:** the failure check intentionally logs
+ `Simulated column catalog failure`. Its result should still be **PASS**.
+
+ This notebook cannot establish real Iceberg network performance. A remote
+ REST catalog with credentials is needed for that measurement; a public CSV
+ would exercise data reads rather than catalog discovery.
+ """)
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.py
index 84f41f2c84..66150a9a93 100644
--- a/marimo/_sql/engines/duckdb.py
+++ b/marimo/_sql/engines/duckdb.py
@@ -5,14 +5,21 @@ from contextlib import contextmanager, nullcontext
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
from marimo import _loggers
-from marimo._data.get_datasets import get_databases_from_duckdb
-from marimo._data.models import Database, DataTable, Schema
+from marimo._data.get_datasets import _db_type_to_data_type, get_table_columns
+from marimo._data.models import (
+ Database,
+ DataTable,
+ DataTableColumn,
+ DataTableType,
+ Schema,
+)
from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime.context.types import (
ContextNotInitializedError,
get_context,
)
from marimo._sql.engines.types import InferenceConfig, SQLConnection
+from marimo._sql.sql_quoting import quote_qualified_name
from marimo._sql.utils import convert_to_output, wrapped_sql
from marimo._types.ids import VariableName
@@ -120,7 +127,6 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
@property
def inference_config(self) -> InferenceConfig:
- # At the moment this isn't being used for duckdb
return InferenceConfig(
auto_discover_schemas=True,
auto_discover_tables="auto",
@@ -166,18 +172,72 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
include_tables: bool | Literal["auto"],
include_table_details: bool | Literal["auto"],
) -> list[Database]:
- """Fetch all databases from the engine. At the moment, will fetch everything."""
- _, _, _ = include_schemas, include_tables, include_table_details
+ """List databases, discovering only the requested metadata depth."""
+ schemas_resolved = self._resolve_should_auto_discover(include_schemas)
+ tables_resolved = self._resolve_should_auto_discover(include_tables)
+ details_resolved = self._resolve_should_auto_discover(
+ include_table_details
+ )
+ # Keep the temporary catalog accessible without scanning its tables.
+ rows = self._query_catalog(
+ "SELECT database_name FROM duckdb_databases() "
+ "WHERE NOT internal OR database_name = 'temp' "
+ "ORDER BY database_name"
+ )
+ databases = {
+ name: Database(
+ name=name,
+ dialect=self.dialect,
+ engine=self._engine_name,
+ schemas_resolved=schemas_resolved,
+ schemas=[],
+ )
+ for (name,) in rows
+ }
+ if schemas_resolved and databases:
+ # duckdb_schemas() scans every catalog before applying filters.
+ # Query it once so remote namespaces are not fetched per database.
+ schema_rows = self._query_catalog(
+ "SELECT database_name, schema_name FROM duckdb_schemas() "
+ "WHERE schema_name NOT IN ('information_schema', 'pg_catalog') "
+ "ORDER BY database_name, schema_name"
+ )
+ for database_name, schema_name in schema_rows:
+ if database_name in databases:
+ databases[database_name].schemas.append(
+ self._make_schema(
+ schema_name,
+ database_name,
+ include_tables=tables_resolved,
+ include_table_details=False,
+ )
+ )
+ if details_resolved:
+ self._load_table_details(
+ {
+ (database.name, schema.name, table.name): table
+ for database in databases.values()
+ for schema in database.schemas
+ for table in schema.tables
+ }
+ )
+ return list(databases.values())
+
+ def _query_catalog(
+ self, query: str, params: list[Any] | None = None
+ ) -> list[Any]:
import duckdb
connection = cast(
duckdb.DuckDBPyConnection, self._connection or duckdb
)
- with self._install_connection(connection):
- return get_databases_from_duckdb(connection, self._engine_name)
+ try:
+ with self._install_connection(connection):
+ return connection.execute(query, params).fetchall()
+ except duckdb.ConnectionException:
+ LOGGER.debug("Skipping closed DuckDB connection")
+ return []
- # TODO: The following methods are currently not implemented.
- # We should consider implementing these in the future for better performance when users don't want to fetch everything.
def get_schemas(
self,
*,
@@ -186,14 +246,46 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
include_table_details: bool,
schema_path: list[str] | None = None,
) -> list[Schema]:
- """Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _, _ = (
- database,
- include_tables,
- include_table_details,
- schema_path,
+ """List schemas without enumerating tables unless requested."""
+ if schema_path:
+ return []
+ if database is None:
+ database = self.get_default_database()
+ if database is None:
+ return []
+ rows = self._query_catalog(
+ "SELECT schema_name FROM duckdb_schemas() "
+ "WHERE database_name = ? "
+ "AND schema_name NOT IN ('information_schema', 'pg_catalog') "
+ "ORDER BY schema_name",
+ [database],
)
- return []
+ return [
+ self._make_schema(
+ name,
+ database,
+ include_tables=include_tables,
+ include_table_details=include_table_details,
+ )
+ for (name,) in rows
+ ]
+
+ def _make_schema(
+ self,
+ name: str,
+ database: str,
+ *,
+ include_tables: bool,
+ include_table_details: bool,
+ ) -> Schema:
+ tables = []
+ if include_tables:
+ tables = self.get_tables_in_schema(
+ schema=name,
+ database=database,
+ include_table_details=include_table_details,
+ )
+ return Schema(name=name, tables=tables, tables_resolved=include_tables)
def get_tables_in_schema(
self,
@@ -203,9 +295,89 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
include_table_details: bool,
schema_path: list[str] | None = None,
) -> list[DataTable]:
- """Return all tables in a schema. This is currently implemented in get_databases_from_duckdb."""
- _, _, _, _ = database, schema, include_table_details, schema_path
- return []
+ """List table and view names without eagerly loading their columns."""
+ del schema_path
+ rows = self._query_catalog(
+ "SELECT table_name, table_type FROM information_schema.tables "
+ "WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",
+ [database, schema],
+ )
+ tables = {
+ (database, schema, name): self._make_table(
+ name,
+ database,
+ table_type="view" if kind == "VIEW" else "table",
+ )
+ for name, kind in rows
+ }
+ if include_table_details:
+ self._load_table_details(tables, database=database, schema=schema)
+ return list(tables.values())
+
+ def _load_table_details(
+ self,
+ tables: dict[tuple[str, str, str], DataTable],
+ *,
+ database: str | None = None,
+ schema: str | None = None,
+ ) -> None:
+ if not tables:
+ return
+ query = (
+ "SELECT database_name, schema_name, table_name, column_name, data_type "
+ "FROM duckdb_columns() WHERE NOT internal"
+ )
+ params: list[str] = []
+ if database is not None:
+ query += " AND database_name = ?"
+ params.append(database)
+ if schema is not None:
+ query += " AND schema_name = ?"
+ params.append(schema)
+ query += (
+ " ORDER BY database_name, schema_name, table_name, column_index"
+ )
+ try:
+ rows = self._query_catalog(query, params)
+ except Exception:
+ # A metadata failure must not hide tables already enumerated.
+ LOGGER.warning("Failed to get DuckDB columns", exc_info=True)
+ return
+ for db_name, schema_name, table_name, column_name, dtype in rows:
+ table = tables.get((db_name, schema_name, table_name))
+ if table is not None:
+ table.columns.append(
+ DataTableColumn(
+ name=column_name,
+ type=_db_type_to_data_type(dtype),
+ external_type=dtype,
+ sample_values=[],
+ )
+ )
+ for (_, schema_name, _), table in tables.items():
+ # Some Iceberg catalogs expose a placeholder instead of columns.
+ if len(table.columns) == 1 and table.columns[0].name == "__":
+ table.columns = []
+ self._describe_table(table, schema_name)
+ if table.columns:
+ table.num_columns = len(table.columns)
+
+ def _make_table(
+ self, name: str, database: str, *, table_type: DataTableType = "table"
+ ) -> DataTable:
+ return DataTable(
+ source_type="duckdb"
+ if self._engine_name is None
+ else "connection",
+ source=database,
+ name=name,
+ type=table_type,
+ num_rows=None,
+ num_columns=None,
+ variable_name=None,
+ columns=[],
+ engine=self._engine_name,
+ )
def get_table_details(
self,
@@ -215,6 +387,31 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
database_name: str,
schema_path: list[str] | None = None,
) -> DataTable | None:
- """Get a single table from the engine. This is currently implemented in get_databases_from_duckdb."""
- _, _, _, _ = table_name, schema_name, database_name, schema_path
- return None
+ """Describe only the requested table, including remote catalog tables."""
+ del schema_path
+ rows = self._query_catalog(
+ "SELECT table_type FROM information_schema.tables "
+ "WHERE table_catalog = ? AND table_schema = ? AND table_name = ?",
+ [database_name, schema_name, table_name],
+ )
+ if not rows:
+ return None
+ table = self._make_table(
+ table_name,
+ database_name,
+ table_type="view" if rows[0][0] == "VIEW" else "table",
+ )
+ self._describe_table(table, schema_name)
+ return table if table.columns else None
+
+ def _describe_table(self, table: DataTable, schema: str) -> None:
+ import duckdb
+
+ connection = cast(
+ duckdb.DuckDBPyConnection, self._connection or duckdb
+ )
+ qualified_name = quote_qualified_name(table.source, schema, table.name)
+ with self._install_connection(connection):
+ table.columns = get_table_columns(connection, qualified_name)
+ if table.columns:
+ table.num_columns = len(table.columns)
diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.py
index 720d0e057c..82e74991a2 100644
--- a/tests/_runtime/test_runtime_datasets.py
+++ b/tests/_runtime/test_runtime_datasets.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import pytest
+from marimo._data.models import DataTable
from marimo._dependencies.dependencies import DependencyManager
from marimo._messaging.notification import (
DataSourceConnectionsNotification,
@@ -24,7 +25,7 @@ from marimo._runtime.commands import (
)
from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE
from marimo._sql.parse import SqlCatalogCheckResult, SqlParseResult
-from marimo._types.ids import CellId_t, RequestId
+from marimo._types.ids import CellId_t, RequestId, VariableName
from tests.conftest import MockedKernel
HAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()
@@ -360,13 +361,21 @@ class TestPreviewSQLTableList:
k = mocked_kernel.k
stream = mocked_kernel.stream
- await k.run(connection_requests)
+ await k.run(
+ [
+ *connection_requests,
+ ExecuteCellCommand(
+ cell_id=CellId_t("4"),
+ code=f"{DUCKDB_CONN}.execute('CREATE TABLE example (id INTEGER)')",
+ ),
+ ]
+ )
preview_sql_table_list_request = ListSQLTablesCommand(
request_id=RequestId("0"),
engine=DUCKDB_CONN,
- database="test",
- schema="test",
+ database="memory",
+ schema="main",
)
await k.handle_message(preview_sql_table_list_request)
@@ -378,10 +387,21 @@ class TestPreviewSQLTableList:
assert preview_sql_table_list_results == [
SQLTableListPreviewNotification(
request_id=RequestId("0"),
- tables=[],
+ tables=[
+ DataTable(
+ name="example",
+ source="memory",
+ source_type="connection",
+ engine=VariableName(DUCKDB_CONN),
+ num_rows=None,
+ num_columns=None,
+ columns=[],
+ variable_name=None,
+ )
+ ],
error=None,
metadata=SQLMetadata(
- connection=DUCKDB_CONN, database="test", schema="test"
+ connection=DUCKDB_CONN, database="memory", schema="main"
),
)
]
diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.py
index f05a8a888a..208971c1f3 100644
--- a/tests/_sql/test_duckdb.py
+++ b/tests/_sql/test_duckdb.py
@@ -4,7 +4,7 @@ from __future__ import annotations
import sys
from copy import deepcopy
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
from unittest import mock
import pytest
@@ -13,7 +13,9 @@ from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._dependencies.dependencies import DependencyManager
from marimo._sql.engines.duckdb import DuckDBEngine
from marimo._sql.engines.types import EngineCatalog, QueryEngine
+from marimo._sql.get_engines import engine_to_data_source_connection
from marimo._sql.sql import sql
+from marimo._sql.sql_quoting import quote_qualified_name
from marimo._types.ids import VariableName
HAS_DUCKDB = DependencyManager.duckdb.has()
@@ -140,7 +142,15 @@ def test_duckdb_engine_get_databases(
include_schemas=True, include_tables=True, include_table_details=True
)
- assert databases == expected_databases_with_conn
+ assert databases == [
+ *expected_databases_with_conn,
+ Database(
+ name="temp",
+ dialect="duckdb",
+ engine=VariableName("test_duckdb"),
+ schemas=[Schema(name="main", tables=[])],
+ ),
+ ]
@pytest.mark.skipif(not HAS_DUCKDB, reason="DuckDB not installed")
@@ -153,7 +163,10 @@ def test_duckdb_engine_get_databases_no_conn() -> None:
include_tables=False,
)
assert initial_databases == [
- Database(name="memory", dialect="duckdb", schemas=[])
+ Database(
+ name=name, dialect="duckdb", schemas=[], schemas_resolved=False
+ )
+ for name in ("memory", "temp")
]
assert engine.get_default_database() == "memory"
assert engine.get_default_schema() == "main"
@@ -179,7 +192,14 @@ def test_duckdb_engine_get_databases_no_conn() -> None:
expected_databases[0].schemas[0].tables[0].source_type = "duckdb"
expected_databases[0].schemas[0].tables[0].source = "memory"
- assert databases == expected_databases
+ assert databases == [
+ *expected_databases,
+ Database(
+ name="temp",
+ dialect="duckdb",
+ schemas=[Schema(name="main", tables=[])],
+ ),
+ ]
engine.execute("DROP TABLE test")
@@ -439,3 +459,381 @@ def test_duckdb_engine_lazy_polars_falls_back_on_older_duckdb(
# Only the first call (lazy=True, raises) reaches `pl`; the fallback uses
# `to_polars()` (Arrow PyCapsule) and never touches `relation.pl()`.
assert pl_calls == [{"batch_size": 100_000, "lazy": True}]
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
[email protected]("include_schemas", [False, True, "auto"])
[email protected]("include_tables", [False, True, "auto"])
[email protected]("include_details", [False, True, "auto"])
+def test_duckdb_discovery_depth(
+ include_schemas: bool | Literal["auto"],
+ include_tables: bool | Literal["auto"],
+ include_details: bool | Literal["auto"],
+) -> None:
+ import duckdb
+
+ with duckdb.connect() as connection:
+ connection.execute("CREATE TABLE test (id INTEGER, name VARCHAR)")
+ engine = DuckDBEngine(connection, VariableName("test_duckdb"))
+ with (
+ mock.patch.object(
+ engine, "_query_catalog", wraps=engine._query_catalog
+ ) as query,
+ mock.patch.object(
+ engine, "get_table_details", wraps=engine.get_table_details
+ ) as details,
+ ):
+ databases = engine.get_databases(
+ include_schemas=include_schemas,
+ include_tables=include_tables,
+ include_table_details=include_details,
+ )
+
+ schemas_enabled = include_schemas is not False
+ tables_enabled = schemas_enabled and include_tables is not False
+ details_enabled = tables_enabled and include_details is not False
+ expected = deepcopy(expected_databases_with_conn[0])
+ if not schemas_enabled:
+ expected.schemas = []
+ expected.schemas_resolved = False
+ elif not tables_enabled:
+ expected.schemas[0].tables = []
+ expected.schemas[0].tables_resolved = False
+ elif not details_enabled:
+ expected.schemas[0].tables[0].columns = []
+ expected.schemas[0].tables[0].num_columns = None
+ assert databases[0] == expected
+ queries = [call.args[0] for call in query.call_args_list]
+ assert sum("duckdb_schemas()" in sql for sql in queries) == int(
+ schemas_enabled
+ )
+ assert (
+ any("information_schema.tables" in sql for sql in queries)
+ == tables_enabled
+ )
+ assert sum("duckdb_columns()" in sql for sql in queries) == int(
+ details_enabled
+ )
+ assert details.call_count == 0
+ assert not any("SHOW ALL TABLES" in sql for sql in queries)
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+def test_duckdb_deferred_discovery() -> None:
+ import duckdb
+
+ database, schema, table_name = 'db".quoted', "schema's.name", 'table".name'
+ qualified_table = quote_qualified_name(database, schema, table_name)
+ with duckdb.connect() as connection:
+ connection.execute(
+ f"ATTACH ':memory:' AS {quote_qualified_name(database)}"
+ )
+ connection.execute(
+ f"CREATE SCHEMA {quote_qualified_name(database, schema)}"
+ )
+ connection.execute(
+ f"CREATE TABLE {qualified_table} (id INTEGER, name VARCHAR)"
+ )
+ connection.execute(
+ f"CREATE VIEW {quote_qualified_name(database, schema, 'a_view')} AS SELECT * FROM {qualified_table}"
+ )
+ connection.execute("CREATE TEMP TABLE local_temp (id INTEGER)")
+ engine = DuckDBEngine(connection, VariableName("test_duckdb"))
+ databases = engine.get_databases(
+ include_schemas=False,
+ include_tables=False,
+ include_table_details=False,
+ )
+ assert {db.name for db in databases} == {database, "memory", "temp"}
+ assert all(
+ not db.schemas_resolved and not db.schemas for db in databases
+ )
+
+ schemas = engine.get_schemas(
+ database=database,
+ include_tables=False,
+ include_table_details=False,
+ )
+ assert schemas == [
+ Schema(name="main", tables=[], tables_resolved=False),
+ Schema(name=schema, tables=[], tables_resolved=False),
+ ]
+ tables = engine.get_tables_in_schema(
+ database=database, schema=schema, include_table_details=False
+ )
+ assert [(table.name, table.type) for table in tables] == [
+ ("a_view", "view"),
+ (table_name, "table"),
+ ]
+ assert all(
+ table.columns == [] and table.num_columns is None
+ for table in tables
+ )
+ expected = deepcopy(
+ expected_databases_with_conn[0].schemas[0].tables[0]
+ )
+ expected.source = database
+ for name in (table_name, "a_view"):
+ expected.name = name
+ expected.type = "view" if name == "a_view" else "table"
+ assert (
+ engine.get_table_details(
+ database_name=database, schema_name=schema, table_name=name
+ )
+ == expected
+ )
+ assert (
+ engine.get_table_details(
+ database_name=database,
+ schema_name=schema,
+ table_name="missing",
+ )
+ is None
+ )
+ assert (
+ engine.get_schemas(
+ database=database,
+ include_tables=False,
+ include_table_details=False,
+ schema_path=[schema],
+ )
+ == []
+ )
+ assert [
+ table.name
+ for table in engine.get_tables_in_schema(
+ database="temp", schema="main", include_table_details=False
+ )
+ ] == ["local_temp"]
+ assert engine.get_schemas(
+ database=None, include_tables=False, include_table_details=False
+ ) == [Schema(name="main", tables=[], tables_resolved=False)]
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
[email protected]("disable_discovery", [False, True])
+def test_duckdb_discovery_config(disable_discovery: bool) -> None:
+ import duckdb
+
+ with duckdb.connect() as connection:
+ connection.execute("CREATE TABLE test (id INTEGER)")
+ name = VariableName("connection")
+ engine = DuckDBEngine(connection, name)
+ config = {}
+ if disable_discovery:
+ config = {
+ "auto_discover_schemas": False,
+ "auto_discover_tables": False,
+ "auto_discover_columns": False,
+ }
+ with mock.patch(
+ "marimo._sql.get_engines.get_datasources_config",
+ return_value=config,
+ ):
+ source = engine_to_data_source_connection(name, engine)
+ if disable_discovery:
+ assert source.databases == [
+ Database(
+ name=db,
+ dialect="duckdb",
+ engine=name,
+ schemas=[],
+ schemas_resolved=False,
+ )
+ for db in ("memory", "temp")
+ ]
+ else:
+ # The default discovers table names but defers columns.
+ assert source.databases[0].schemas[0].tables == [
+ DataTable(
+ name="test",
+ source="memory",
+ source_type="connection",
+ engine=name,
+ num_rows=None,
+ num_columns=None,
+ columns=[],
+ variable_name=None,
+ )
+ ]
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+def test_duckdb_discovery_closed_connection() -> None:
+ import duckdb
+
+ connection = duckdb.connect()
+ connection.close()
+ engine = DuckDBEngine(connection)
+ assert (
+ engine.get_databases(
+ include_schemas=True,
+ include_tables=True,
+ include_table_details=True,
+ )
+ == []
+ )
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+def test_duckdb_discovers_schemas_once_for_all_databases() -> None:
+ import duckdb
+
+ with duckdb.connect() as connection:
+ for name in ("first", "second"):
+ connection.execute(f"ATTACH ':memory:' AS {name}")
+ connection.execute(f"CREATE SCHEMA {name}.extra")
+ engine = DuckDBEngine(connection)
+ with mock.patch.object(
+ engine, "_query_catalog", wraps=engine._query_catalog
+ ) as query:
+ databases = engine.get_databases(
+ include_schemas=True,
+ include_tables=False,
+ include_table_details=False,
+ )
+ assert databases == [
+ Database(
+ name=name,
+ dialect="duckdb",
+ schemas=[
+ Schema(name=schema, tables=[], tables_resolved=False)
+ for schema in schemas
+ ],
+ )
+ for name, schemas in [
+ ("first", ["extra", "main"]),
+ ("memory", ["main"]),
+ ("second", ["extra", "main"]),
+ ("temp", ["main"]),
+ ]
+ ]
+ assert len(query.call_args_list) == 2
+ assert (
+ sum(
+ "duckdb_schemas()" in call.args[0]
+ for call in query.call_args_list
+ )
+ == 1
+ )
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
[email protected]("all_databases", [False, True])
+def test_duckdb_batches_column_discovery(all_databases: bool) -> None:
+ import duckdb
+
+ with duckdb.connect() as connection:
+ connection.execute("ATTACH ':memory:' AS attached")
+ for database in ("memory", "attached"):
+ connection.execute(f"CREATE SCHEMA {database}.extra")
+ for schema in ("main", "extra"):
+ connection.execute(
+ f"CREATE TABLE {database}.{schema}.example (z INTEGER, a VARCHAR)"
+ )
+ connection.execute(
+ f"CREATE VIEW {database}.{schema}.example_view AS SELECT * FROM {database}.{schema}.example"
+ )
+ engine = DuckDBEngine(connection)
+ with (
+ mock.patch.object(
+ engine, "_query_catalog", wraps=engine._query_catalog
+ ) as query,
+ mock.patch.object(
+ engine, "_describe_table", wraps=engine._describe_table
+ ) as describe,
+ ):
+ if all_databases:
+ databases = engine.get_databases(
+ include_schemas=True,
+ include_tables=True,
+ include_table_details=True,
+ )
+ tables = [
+ table
+ for database in databases
+ for schema in database.schemas
+ for table in schema.tables
+ ]
+ else:
+ tables = engine.get_tables_in_schema(
+ database="attached",
+ schema="main",
+ include_table_details=True,
+ )
+ assert len(tables) == (8 if all_databases else 2)
+ for table in tables:
+ assert table.columns == [
+ DataTableColumn(
+ name="z",
+ type="integer",
+ external_type="INTEGER",
+ sample_values=[],
+ ),
+ DataTableColumn(
+ name="a",
+ type="string",
+ external_type="VARCHAR",
+ sample_values=[],
+ ),
+ ]
+ assert table.num_columns == 2
+ assert table.type == (
+ "view" if table.name == "example_view" else "table"
+ )
+ assert (
+ sum(
+ "duckdb_columns()" in call.args[0]
+ for call in query.call_args_list
+ )
+ == 1
+ )
+ describe.assert_not_called()
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
[email protected]("failure", ["query", "missing", "placeholder"])
+def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:
+ import duckdb
+
+ with duckdb.connect() as connection:
+ connection.execute("CREATE TABLE example (id INTEGER)")
+ engine = DuckDBEngine(connection)
+ query = engine._query_catalog
+
+ def failing_columns(
+ sql: str, params: list[Any] | None = None
+ ) -> list[Any]:
+ if "duckdb_columns()" in sql:
+ if failure == "query":
+ raise duckdb.NotImplementedException(
+ "Column metadata unavailable"
+ )
+ if failure == "placeholder":
+ return [("memory", "main", "example", "__", "INTEGER")]
+ return []
+ return query(sql, params)
+
+ with (
+ mock.patch.object(
+ engine, "_query_catalog", side_effect=failing_columns
+ ),
+ mock.patch(
+ "marimo._sql.engines.duckdb.get_table_columns", return_value=[]
+ ) as describe,
+ ):
+ tables = engine.get_tables_in_schema(
+ database="memory", schema="main", include_table_details=True
+ )
+ assert tables == [
+ DataTable(
+ name="example",
+ source="memory",
+ source_type="duckdb",
+ num_rows=None,
+ num_columns=None,
+ variable_name=None,
+ columns=[],
+ )
+ ]
+ assert describe.call_count == int(failure == "placeholder")
diff --git a/tests/_sql/test_get_engines.py b/tests/_sql/test_get_engines.py
index b202f0f6bb..9f91469adf 100644
--- a/tests/_sql/test_get_engines.py
+++ b/tests/_sql/test_get_engines.py
@@ -8,7 +8,6 @@ import pytest
from marimo._data.models import (
Database,
DataSourceConnection,
- DataTableColumn,
Schema,
)
from marimo._dependencies.dependencies import DependencyManager
@@ -52,7 +51,12 @@ def test_engine_to_data_source_connection() -> None:
assert connection.default_database == "memory"
assert connection.default_schema == "main"
assert connection.databases == [
- Database(name="memory", dialect="duckdb", schemas=[])
+ Database(
+ name=name,
+ dialect="duckdb",
+ schemas=[Schema(name="main", tables=[])],
+ )
+ for name in ("memory", "temp")
]
# Test with ClickhouseEmbedded engine
@@ -357,7 +361,10 @@ def test_get_engines_duckdb_databases() -> None:
VariableName("my_duckdb"), duckdb_engine
)
- assert len(connection.databases) == 1
+ assert [database.name for database in connection.databases] == [
+ "memory",
+ "temp",
+ ]
database = connection.databases[0]
assert database.name == "memory"
assert len(database.schemas) == 1
@@ -366,14 +373,8 @@ def test_get_engines_duckdb_databases() -> None:
assert len(schema.tables) == 1
table = schema.tables[0]
assert table.name == "test_table"
- assert table.columns == [
- DataTableColumn(
- name="id",
- type="integer",
- external_type="INTEGER",
- sample_values=[],
- )
- ]
+ assert table.columns == []
+ assert table.num_columns is None
with patch(
"marimo._sql.sql.get_configured_sql_output_format",
return_value="native",