Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

new query to get duckdb databases for ducklakes (#6708)

Shahmir Varqha873cdc6ff04a
diff --git a/marimo/_data/get_datasets.py b/marimo/_data/get_datasets.py
index 322e44e4eb..04da219334 100644
--- a/marimo/_data/get_datasets.py
+++ b/marimo/_data/get_datasets.py
@@ -11,6 +11,7 @@ from marimo._data.models import (
DataType,
Schema,
)
+from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins.ui._impl.tables.utils import get_table_manager_or_none
from marimo._types.ids import VariableName
@@ -98,7 +99,7 @@ def execute_duckdb_query(
return connection.execute(query).fetchall()
except Exception:
- LOGGER.exception("Failed to execute DuckDB query")
+ LOGGER.exception("Failed to execute DuckDB query %s", query)
return []
@@ -113,6 +114,23 @@ def get_databases_from_duckdb(
return []
+def _get_empty_databases(
+ connection: Optional[duckdb.DuckDBPyConnection],
+ engine_name: Optional[VariableName],
+) -> list[Database]:
+ # Fallback to get database names from DuckDB
+ all_dbs = _get_duckdb_database_names(connection)
+ return [
+ Database(
+ name=database,
+ dialect="duckdb",
+ schemas=[],
+ engine=engine_name,
+ )
+ for database in all_dbs
+ ]
+
+
def _get_databases_from_duckdb_internal(
connection: Optional[duckdb.DuckDBPyConnection],
engine_name: Optional[VariableName] = None,
@@ -125,20 +143,28 @@ def _get_databases_from_duckdb_internal(
# 3:"column_names"
# 4:"column_types"
# 5:"temporary"
- tables_result = execute_duckdb_query(connection, "SHOW ALL TABLES")
+ tables_result = []
+ query = "SHOW ALL TABLES"
+ try:
+ if connection is None:
+ import duckdb
+
+ tables_result = duckdb.execute(query).fetchall()
+ else:
+ tables_result = connection.execute(query).fetchall()
+ except Exception as e:
+ if DependencyManager.duckdb.has():
+ import duckdb
+
+ # Certain ducklakes don't support SHOW ALL TABLES
+ if isinstance(e, duckdb.NotImplementedException):
+ return get_duckdb_databases_agg_query(connection, engine_name)
+
+ LOGGER.exception("Failed to get tables from DuckDB")
+ return []
if len(tables_result) == 0:
- # Return empty databases if there are no tables
- all_dbs = _get_duckdb_database_names(connection)
- return [
- Database(
- name=database,
- dialect="duckdb",
- schemas=[],
- engine=engine_name,
- )
- for database in all_dbs
- ]
+ return _get_empty_databases(connection, engine_name)
# Group tables by database and schema
# databases_dict[database][schema] = [table1, table2, ...]
@@ -204,34 +230,7 @@ def _get_databases_from_duckdb_internal(
databases_dict[database][schema].append(table)
- # Convert grouped data into Database objects
- databases: list[Database] = []
- for database, schemas_dict in databases_dict.items():
- schema_list: list[Schema] = []
- for schema_name, tables in schemas_dict.items():
- schema_list.append(Schema(name=schema_name, tables=tables))
- databases.append(
- Database(
- name=database,
- dialect="duckdb",
- schemas=schema_list,
- engine=engine_name,
- )
- )
-
- # There may be remaining databases not surfaced with SHOW ALL TABLES
- # These db's likely have no tables
- for database_name in _get_duckdb_database_names(connection):
- if database_name not in databases_dict:
- databases.append(
- Database(
- name=database_name,
- dialect="duckdb",
- schemas=[],
- engine=engine_name,
- )
- )
- return databases
+ return form_databases_from_dict(databases_dict, connection, engine_name)
def get_table_columns(
@@ -269,6 +268,127 @@ def get_table_columns(
return []
+def form_databases_from_dict(
+ databases_dict: dict[str, dict[str, list[DataTable]]],
+ connection: Optional[duckdb.DuckDBPyConnection],
+ engine_name: Optional[VariableName],
+) -> list[Database]:
+ # Convert grouped data into Database objects
+ databases: list[Database] = []
+ for database, schemas_dict in databases_dict.items():
+ schema_list: list[Schema] = []
+ for schema_name, tables in schemas_dict.items():
+ schema_list.append(Schema(name=schema_name, tables=tables))
+ databases.append(
+ Database(
+ name=database,
+ dialect="duckdb",
+ schemas=schema_list,
+ engine=engine_name,
+ )
+ )
+
+ # There may be remaining databases not surfaced with SHOW ALL TABLES
+ # These db's likely have no tables
+ for database_name in _get_duckdb_database_names(connection):
+ if database_name not in databases_dict:
+ databases.append(
+ Database(
+ name=database_name,
+ dialect="duckdb",
+ schemas=[],
+ engine=engine_name,
+ )
+ )
+ return databases
+
+
+def get_duckdb_databases_agg_query(
+ connection: Optional[duckdb.DuckDBPyConnection],
+ engine_name: Optional[VariableName],
+) -> list[Database]:
+ """Uses a different query to get database information, which has wider support but has some aggregation overhead"""
+
+ # Cols will be in the form of [{"col": "column_name", "dtype": "data_type"}]
+ NAME_KEY = "col"
+ DTYPE_KEY = "dtype"
+
+ QUERY = f"""
+ SELECT
+ database_name,
+ schema_name,
+ table_name,
+ ARRAY_AGG(
+ struct_pack({NAME_KEY} := column_name, {DTYPE_KEY} := data_type)
+ ORDER BY
+ column_index
+ ) AS cols
+ FROM
+ duckdb_columns()
+ WHERE
+ internal = false
+ AND table_name NOT IN ('duckdb_functions()', 'duckdb_types()', 'duckdb_settings()')
+ GROUP BY
+ database_name,
+ schema_name,
+ table_name
+ ORDER BY database_name, schema_name, table_name
+ """
+
+ tables_result = execute_duckdb_query(connection, QUERY)
+ if len(tables_result) == 0:
+ return _get_empty_databases(connection, engine_name)
+
+ # Group tables by database and schema
+ # databases_dict[database][schema] = [table1, table2, ...]
+ databases_dict: dict[str, dict[str, list[DataTable]]] = {}
+
+ for (
+ database_name,
+ schema_name,
+ table_name,
+ cols,
+ ) in tables_result:
+ columns: list[DataTableColumn] = []
+ assert isinstance(cols, list)
+ for col in cols:
+ assert isinstance(col, dict)
+ assert NAME_KEY in col
+ assert DTYPE_KEY in col
+ dtype = col[DTYPE_KEY]
+ columns.append(
+ DataTableColumn(
+ name=col[NAME_KEY],
+ type=_db_type_to_data_type(dtype),
+ external_type=dtype,
+ sample_values=[],
+ )
+ )
+
+ table = DataTable(
+ name=table_name,
+ columns=columns,
+ source_type="duckdb" if engine_name is None else "connection",
+ source=database_name,
+ num_rows=None,
+ num_columns=len(columns),
+ variable_name=None,
+ engine=engine_name,
+ type="table",
+ primary_keys=None,
+ indexes=None,
+ )
+
+ if database_name not in databases_dict:
+ databases_dict[database_name] = {}
+ if schema_name not in databases_dict[database_name]:
+ databases_dict[database_name][schema_name] = []
+
+ databases_dict[database_name][schema_name].append(table)
+
+ return form_databases_from_dict(databases_dict, connection, engine_name)
+
+
def _get_duckdb_database_names(
connection: Optional[duckdb.DuckDBPyConnection],
) -> list[str]:
diff --git a/tests/_data/test_get_datasets.py b/tests/_data/test_get_datasets.py
index 4dd659d87c..601158e83f 100644
--- a/tests/_data/test_get_datasets.py
+++ b/tests/_data/test_get_datasets.py
@@ -8,11 +8,13 @@ import pytest
from marimo._data.get_datasets import (
get_databases_from_duckdb,
get_datasets_from_variables,
+ get_duckdb_databases_agg_query,
get_table_columns,
has_updates_to_datasource,
)
from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._dependencies.dependencies import DependencyManager
+from marimo._types.ids import VariableName
from tests._data.mocks import create_dataframes
HAS_DEPS = DependencyManager.duckdb.has()
@@ -425,12 +427,18 @@ def test_get_databases_with_no_tables() -> None:
assert get_databases_from_duckdb(connection=connection) == [
in_memory_database
]
+ assert get_duckdb_databases_agg_query(
+ connection=connection, engine_name=None
+ ) == [in_memory_database]
# Custom connection with no tables
connection = duckdb.connect(":memory:")
assert get_databases_from_duckdb(connection=connection) == [
in_memory_database
]
+ assert get_duckdb_databases_agg_query(
+ connection=connection, engine_name=None
+ ) == [in_memory_database]
@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed")
@@ -443,21 +451,19 @@ def test_get_databases_with_connection() -> None:
all_tables = deepcopy(all_types_tables)
for table in all_tables:
table.source_type = "connection"
- table.engine = "engine"
+ table.engine = VariableName("engine")
s1 = deepcopy(s1_tables)
for table in s1:
table.source_type = "connection"
- table.engine = "engine"
+ table.engine = VariableName("engine")
s2 = deepcopy(s2_tables)
for table in s2:
table.source_type = "connection"
- table.engine = "engine"
+ table.engine = VariableName("engine")
- assert get_databases_from_duckdb(
- connection=connection, engine_name="engine"
- ) == [
+ expected_databases = [
Database(
name="memory",
dialect="duckdb",
@@ -466,10 +472,24 @@ def test_get_databases_with_connection() -> None:
Schema(name="s1", tables=s1),
Schema(name="s2", tables=s2),
],
- engine="engine",
+ engine=VariableName("engine"),
)
]
+ assert (
+ get_databases_from_duckdb(
+ connection=connection, engine_name=VariableName("engine")
+ )
+ == expected_databases
+ )
+
+ assert (
+ get_duckdb_databases_agg_query(
+ connection=connection, engine_name=VariableName("engine")
+ )
+ == expected_databases
+ )
+
connection.execute(cleanup_query)
@@ -478,7 +498,9 @@ def test_get_databases_with_connection() -> None:
create_dataframes({"A": [1, 2, 3], "B": ["a", "a", "a"]}),
)
def test_get_datasets_from_variables(df: Any) -> None:
- datatests = get_datasets_from_variables([("my_df", df), ("non_df", 123)])
+ datatests = get_datasets_from_variables(
+ [(VariableName("my_df"), df), (VariableName("non_df"), 123)]
+ )
# We don't compare these values
external_type1 = datatests[0].columns[0].external_type
external_type2 = datatests[0].columns[1].external_type
@@ -498,7 +520,7 @@ def test_get_datasets_from_variables(df: Any) -> None:
source="memory",
num_rows=rows,
num_columns=2,
- variable_name="my_df",
+ variable_name=VariableName("my_df"),
columns=[
DataTableColumn(
name="A",
@@ -525,3 +547,27 @@ def test_get_table_columns() -> None:
columns = get_table_columns(connection, "all_types")
assert columns == all_types_tables[0].columns
+
+
+def test_get_databases_agg_query() -> None:
+ import duckdb
+
+ connection = duckdb.connect(":memory:")
+ connection.execute(sql_query)
+
+ assert get_duckdb_databases_agg_query(
+ connection=connection, engine_name=None
+ ) == [
+ Database(
+ name="memory",
+ dialect="duckdb",
+ schemas=[
+ Schema(name="main", tables=all_types_tables),
+ Schema(name="s1", tables=s1_tables),
+ Schema(name="s2", tables=s2_tables),
+ ],
+ engine=None,
+ )
+ ]
+
+ connection.execute(cleanup_query)