A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(sql): preserve nested schema paths in datasource browsing (#11458)
Shahmir Varqha0423673a73e9
diff --git a/frontend/src/components/datasources/__tests__/column-preview.test.tsx b/frontend/src/components/datasources/__tests__/column-preview.test.tsxindex c6f0784339..0202d328f6 100644--- a/frontend/src/components/datasources/__tests__/column-preview.test.tsx+++ b/frontend/src/components/datasources/__tests__/column-preview.test.tsx@@ -79,7 +79,9 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {it("qualifies with database and schema for flat engines", () => {const client = renderPreview(ctx({ database: "db", schema: "public" }));expect(client.previewDatasetColumn).toHaveBeenCalledWith(- expect.objectContaining({ fullyQualifiedTableName: "db.public.users" }),+ expect.objectContaining({+ fullyQualifiedTableName: '"db"."public"."users"',+ }),);});@@ -87,7 +89,7 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {// Regression: previously produced "db..users".const client = renderPreview(ctx({ database: "db", schema: "" }));expect(client.previewDatasetColumn).toHaveBeenCalledWith(- expect.objectContaining({ fullyQualifiedTableName: "db.users" }),+ expect.objectContaining({ fullyQualifiedTableName: '"db"."users"' }),);});@@ -98,7 +100,37 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {);expect(client.previewDatasetColumn).toHaveBeenCalledWith(expect.objectContaining({- fullyQualifiedTableName: "top.nested.deep.users",+ fullyQualifiedTableName: '"top"."nested"."deep"."users"',+ }),+ );+ });++ it("quotes a literal dot as one schema segment", () => {+ const client = renderPreview(+ ctx({+ database: "memory",+ schema: "finance.reports",+ schemaPath: ["finance.reports"],+ }),+ );+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(+ expect.objectContaining({+ fullyQualifiedTableName: '"memory"."finance.reports"."users"',+ }),+ );+ });++ it("escapes quotes inside schema identifiers", () => {+ const client = renderPreview(+ ctx({+ database: "memory",+ schema: 'fi"nance',+ schemaPath: ['fi"nance', "reports"],+ }),+ );+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(+ expect.objectContaining({+ fullyQualifiedTableName: '"memory"."fi""nance"."reports"."users"',}),);});diff --git a/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsx b/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsxnew file mode 100644index 0000000000..3032406430--- /dev/null+++ b/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsx@@ -0,0 +1,181 @@+/* Copyright 2026 Marimo. All rights reserved. */++import {+ act,+ fireEvent,+ render,+ screen,+ waitFor,+} from "@testing-library/react";+import { createStore, Provider } from "jotai";+import { beforeAll, describe, expect, it, vi } from "vitest";+import { MockRequestClient } from "@/__mocks__/requests";+import { Command, CommandList } from "@/components/ui/command";+import { TooltipProvider } from "@/components/ui/tooltip";+import {+ dataSourceConnectionsAtom,+ type SQLTableContext,+} from "@/core/datasets/data-source-connections";+import { DUCKDB_ENGINE } from "@/core/datasets/engines";+import { datasetsAtom, exportedForTesting } from "@/core/datasets/state";+import type { DataTable } from "@/core/kernel/messages";+import { requestClientAtom } from "@/core/network/requests";+import {+ DataSources,+ hideEmptyDatasourcesAtom,+ TableList,+} from "../datasources";++beforeAll(() => {+ HTMLElement.prototype.scrollIntoView = vi.fn();+});++const table: DataTable = {+ name: "reports",+ columns: [+ {+ name: "amount",+ type: "integer",+ external_type: "INTEGER",+ sample_values: [],+ },+ ],+ source: "memory",+ source_type: "duckdb",+ type: "table",+ engine: null,+ indexes: null,+ num_columns: 1,+ num_rows: null,+ variable_name: null,+ primary_keys: null,+};++describe("Datasource column preview responses", () => {+ it.each([+ { path: ["main"], qualifiedName: '"memory"."main"."reports"' },+ {+ path: ["finance", "reports"],+ qualifiedName: '"memory"."finance"."reports"."reports"',+ },+ {+ path: ["finance.reports"],+ qualifiedName: '"memory"."finance.reports"."reports"',+ },+ {+ path: ['fi"nance', "reports"],+ qualifiedName: '"memory"."fi""nance"."reports"."reports"',+ },+ ])(+ "renders the returned preview for $path",+ async ({ path, qualifiedName }) => {+ const store = createStore();+ const client = MockRequestClient.create();+ store.set(requestClientAtom, client);+ const context: SQLTableContext = {+ engine: DUCKDB_ENGINE,+ database: "memory",+ schema: path.at(-1) ?? "",+ schemaPath: path,+ dialect: "duckdb",+ };+ render(+ <Provider store={store}>+ <TooltipProvider>+ <Command shouldFilter={false}>+ <CommandList>+ <TableList tables={[table]} sqlTableContext={context} />+ </CommandList>+ </Command>+ </TooltipProvider>+ </Provider>,+ );++ fireEvent.click(await screen.findByText("reports"));+ fireEvent.click(await screen.findByText("amount"));+ await waitFor(() => {+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(+ expect.objectContaining({ fullyQualifiedTableName: qualifiedName }),+ );+ });+ expect(screen.getByText("Loading...")).toBeInTheDocument();++ const actions = exportedForTesting.createActions((action) => {+ store.set(datasetsAtom, (state) =>+ exportedForTesting.reducer(state, action),+ );+ });+ act(() => {+ actions.addColumnPreview({+ table_name: qualifiedName,+ column_name: "amount",+ stats: { min: 42 },+ error: null,+ missing_packages: null,+ chart_spec: null,+ chart_code: null,+ });+ });++ expect(await screen.findByText("42")).toBeInTheDocument();+ expect(screen.queryByText("Loading...")).not.toBeInTheDocument();+ },+ );+});++it("activates a schema and a table with the same path independently using the keyboard", async () => {+ const store = createStore();+ store.set(requestClientAtom, MockRequestClient.create());+ store.set(hideEmptyDatasourcesAtom, false);+ store.set(dataSourceConnectionsAtom, {+ latestEngineSelected: DUCKDB_ENGINE,+ connectionsMap: new Map([+ [+ DUCKDB_ENGINE,+ {+ name: DUCKDB_ENGINE,+ dialect: "duckdb",+ source: "duckdb",+ display_name: "DuckDB (In-Memory)",+ databases: [+ {+ name: "memory",+ dialect: "duckdb",+ schemas: [+ {+ name: "finance",+ tables: [table],+ child_schemas: [{ name: "reports", tables: [] }],+ },+ ],+ },+ ],+ },+ ],+ ]),+ });+ render(+ <Provider store={store}>+ <TooltipProvider>+ <DataSources />+ </TooltipProvider>+ </Provider>,+ );+ fireEvent.click(await screen.findByText("memory"));+ fireEvent.click(await screen.findByText("finance"));+ await waitFor(() => {+ expect(screen.getAllByText("reports")).toHaveLength(2);+ });++ const input = screen.getByRole("combobox");+ input.focus();+ fireEvent.keyDown(input, { key: "ArrowDown" });+ fireEvent.keyDown(input, { key: "Enter" });+ expect(await screen.findByText("No tables found")).toBeInTheDocument();+ expect(screen.queryByText("amount")).not.toBeInTheDocument();++ fireEvent.keyDown(input, { key: "ArrowDown" });+ fireEvent.keyDown(input, { key: "Enter" });+ expect(await screen.findByText("amount")).toBeInTheDocument();+ expect(screen.getByText("No tables found")).toBeInTheDocument();+});diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.tsindex bad765cf5b..f2d2957688 100644--- a/frontend/src/components/datasources/__tests__/utils.test.ts+++ b/frontend/src/components/datasources/__tests__/utils.test.ts@@ -34,6 +34,37 @@ describe("sqlCode", () => {name: "email" as const,} as DataTableColumn;+ it.each([+ {+ path: ["finance", "reports"],+ expected: '"memory"."finance"."reports"."users"',+ },+ { path: ["finance.reports"], expected: '"finance.reports"."users"' },+ {+ path: ['fi"nance', "reports"],+ expected: '"memory"."fi""nance"."reports"."users"',+ },+ {+ path: ["main", "reports"],+ expected: '"memory"."main"."reports"."users"',+ },+ ])("preserves schema path $path in generated SQL", ({ path, expected }) => {+ const result = sqlCode({+ table: mockTable,+ columnName: "*",+ sqlTableContext: {+ engine: DUCKDB_ENGINE,+ dialect: "duckdb",+ database: "memory",+ defaultDatabase: "memory",+ defaultSchema: "main",+ schema: path.at(-1)!,+ schemaPath: path,+ },+ });+ expect(result).toContain(`FROM ${expected}`);+ });+describe("basic SQL generation", () => {it("should generate basic SQL without sqlTableContext", () => {const result = sqlCode({ table: mockTable, columnName: mockColumn.name });@@ -484,7 +515,7 @@ describe("tableUniqueId", () => {it("uses database + schema for flat engines", () => {expect(tableUniqueId(ctx({ database: "db", schema: "public" }), "t")).toBe(- "db.public.t",+ '["db","public","t"]',);});@@ -495,7 +526,7 @@ describe("tableUniqueId", () => {ctx({ database: "top", schema: "nested", schemaPath: ["nested"] }),"t",),- ).toBe("top.nested.t");+ ).toBe('["top","nested","t"]');});it("includes the full schema path for deeply nested namespaces", () => {@@ -508,7 +539,7 @@ describe("tableUniqueId", () => {}),"t",),- ).toBe("top.nested.deep.t");+ ).toBe('["top","nested","deep","t"]');});it("falls back to the flat schema when schemaPath is empty", () => {@@ -517,16 +548,36 @@ describe("tableUniqueId", () => {ctx({ database: "db", schema: "public", schemaPath: [] }),"t",),- ).toBe("db.public.t");+ ).toBe('["db","public","t"]');});it("does not emit a double dot for schemaless tables", () => {expect(tableUniqueId(ctx({ database: "db", schema: "", schemaPath: [] }), "t"),- ).toBe("db.t");+ ).toBe('["db","t"]');expect(tableUniqueId(ctx({ database: "db", schema: "" }), "t")).toBe(- "db.t",+ '["db","t"]',+ );+ });++ it("keeps literal-dot and nested schemas distinct", () => {+ const nested = tableUniqueId(+ ctx({+ database: "memory",+ schema: "reports",+ schemaPath: ["finance", "reports"],+ }),+ "sales",+ );+ const literal = tableUniqueId(+ ctx({+ database: "memory",+ schema: "finance.reports",+ schemaPath: ["finance.reports"],+ }),+ "sales",);+ expect(nested).not.toBe(literal);});});diff --git a/frontend/src/components/datasources/column-preview.tsx b/frontend/src/components/datasources/column-preview.tsxindex d68fd24302..ce85747d42 100644--- a/frontend/src/components/datasources/column-preview.tsx+++ b/frontend/src/components/datasources/column-preview.tsx@@ -29,7 +29,7 @@ import { Button } from "../ui/button";import { Tooltip } from "../ui/tooltip";import { ColumnPreviewContainer } from "./components";import { InstallPackageButton } from "./install-package-button";-import { convertStatsName, sqlCode, tableUniqueId } from "./utils";+import { convertStatsName, fullyQualifiedTableName, sqlCode } from "./utils";export const DatasetColumnPreview: React.FC<{table: DataTable;@@ -48,7 +48,10 @@ export const DatasetColumnPreview: React.FC<{tableName: table.name,columnName: column.name,sourceType: table.source_type,- fullyQualifiedTableName: tableUniqueId(sqlTableContext, table.name),+ fullyQualifiedTableName: fullyQualifiedTableName(+ sqlTableContext,+ table.name,+ ),});};diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsxindex c18af29e64..b5060169ed 100644--- a/frontend/src/components/datasources/datasources.tsx+++ b/frontend/src/components/datasources/datasources.tsx@@ -78,6 +78,7 @@ import {areChildSchemasResolved,areSchemasResolved,areTablesResolved,+ fullyQualifiedTableName,isSchemaless,schemaSubtreeMatchesSearch,shouldExpandDatabaseForSearch,@@ -637,7 +638,7 @@ const SchemaNode: React.FC<SchemaNodeProps> = (props) => {const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);const [prevExpandForSearch, setPrevExpandForSearch] =React.useState(expandForSearch);- const uniqueValue = `${databaseName}:${schemaPath.join(".")}`;+ const uniqueValue = JSON.stringify(["schema", databaseName, ...schemaPath]);const childSchemas = schema.child_schemas ?? [];// Re-sync when the query changes or when newly resolved data turns this@@ -1095,9 +1096,7 @@ const DatasetColumnItem: React.FC<{column={column}onAddColumnChart={handleAddColumn}preview={columnsPreviews.get(- sqlTableContext- ? `${sqlTableContext.database}.${sqlTableContext.schema}.${table.name}:${column.name}`- : `${table.name}:${column.name}`,+ `${fullyQualifiedTableName(sqlTableContext, table.name)}:${column.name}`,)}sqlTableContext={sqlTableContext}/>diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.tsindex 904d152acb..4a9d5906ec 100644--- a/frontend/src/components/datasources/utils.ts+++ b/frontend/src/components/datasources/utils.ts@@ -13,26 +13,29 @@ import type {import { logNever } from "@/utils/assertNever";import type { ColumnHeaderStatsKey } from "../data-table/types";-/**- * Stable id for a table node in the datasources tree.- *- * schemaPath already includes the leaf schema for nested namespaces, so use it- * when present and fall back to the flat schema name otherwise (avoids- * duplicating the leaf, e.g. `top.nested.nested.table`).- */+function tablePath(context: SQLTableContext, tableName: string): string[] {+ const segments = context.schemaPath?.length+ ? context.schemaPath+ : [context.schema];+ return [context.database, ...segments.filter(Boolean), tableName];+}+export function tableUniqueId(- sqlTableContext: SQLTableContext | undefined,+ context: SQLTableContext | undefined,tableName: string,): string {- if (!sqlTableContext) {- return tableName;- }- const segments = (- sqlTableContext.schemaPath?.length- ? sqlTableContext.schemaPath- : [sqlTableContext.schema]- ).filter(Boolean);- return [sqlTableContext.database, ...segments, tableName].join(".");+ return context ? JSON.stringify(tablePath(context, tableName)) : tableName;+}++export function fullyQualifiedTableName(+ context: SQLTableContext | undefined,+ tableName: string,+): string {+ return context+ ? getFormatter(context.dialect).formatTablePath(+ tablePath(context, tableName),+ )+ : tableName;}// Some databases have no schemas, so we don't show it (eg. Clickhouse)@@ -114,6 +117,10 @@ const defaultFormatter: SqlCodeFormatter = {`SELECT ${columnName} FROM ${tableName} LIMIT 100`,};+function quoteIdentifier(identifier: string): string {+ return `"${identifier.replaceAll('"', '""')}"`;+}+function getFormatter(dialect: string): SqlCodeFormatter {dialect = dialect.toLowerCase();if (!isKnownDialect(dialect)) {@@ -146,9 +153,9 @@ function getFormatter(dialect: string): SqlCodeFormatter {// Quote column and table names to avoid raising errors on weird charactersreturn {formatTablePath: (tablePath: string[]) =>- tablePath.map((part) => `"${part}"`).join("."),+ tablePath.map(quoteIdentifier).join("."),formatSelectClause: (columnName: string, tableName: string) =>- `SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,+ `SELECT ${columnName === "*" ? "*" : quoteIdentifier(columnName)} FROM ${tableName} LIMIT 100`,};case "db2":case "db2i":@@ -194,6 +201,7 @@ export function sqlCode({engine,schema,defaultSchema,+ schemaPath,defaultDatabase,database,dialect,@@ -207,12 +215,13 @@ export function sqlCode({}} else {// Include schema if it's not the default schema- if (schema !== defaultSchema) {- tablePath.unshift(schema);+ const path = schemaPath?.length ? schemaPath : [schema];+ if (path.length > 1 || path[0] !== defaultSchema) {+ tablePath.unshift(...path);}- // Include database if it's not the default database- if (database !== defaultDatabase) {+ // Nested schema names can also be attached catalog names.+ if (path.length > 1 || database !== defaultDatabase) {tablePath.unshift(database);}}diff --git a/marimo/_data/sql_summaries.py b/marimo/_data/sql_summaries.pyindex b7c7f63fab..92d891a06e 100644--- a/marimo/_data/sql_summaries.py+++ b/marimo/_data/sql_summaries.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom marimo._data.get_datasets import _db_type_to_data_typefrom marimo._data.models import ColumnStats, DataTypefrom marimo._sql.sql_quoting import parse_fully_qualified_table_name-from marimo._sql.utils import execute_duckdb_sql, wrapped_sql+from marimo._sql.utils import execute_duckdb_sql, is_duckdb_v2, wrapped_sqldef get_sql_stats(@@ -146,8 +146,14 @@ def get_column_type(Get the type of a column in a SQL table."""- # First, get the column info and data type- if "." in fully_qualified_table_name:+ # DuckDB 1.0's pragma_table_info does not preserve escaped identifier quotes.+ if is_duckdb_v2():+ table_name = fully_qualified_table_name+ column_info_result: tuple[str] | None = execute_duckdb_sql(+ "SELECT type FROM pragma_table_info(?) WHERE name = ?",+ [fully_qualified_table_name, column_name],+ ).fetchone()+ elif "." in fully_qualified_table_name:# Fully qualified table namedb_name, schema_name, table_name = parse_fully_qualified_table_name(fully_qualified_table_name@@ -160,7 +166,7 @@ def get_column_type(AND table_catalog = $3AND column_name = $4"""- column_info_result: tuple[str] | None = execute_duckdb_sql(+ column_info_result = execute_duckdb_sql(column_info_query,[table_name, schema_name, db_name, column_name],).fetchone()diff --git a/marimo/_runtime/callbacks/datasets.py b/marimo/_runtime/callbacks/datasets.pyindex 4f32a96336..d9fd2d848b 100644--- a/marimo/_runtime/callbacks/datasets.py+++ b/marimo/_runtime/callbacks/datasets.py@@ -29,6 +29,7 @@ from marimo._runtime.commands import (PreviewDatasetColumnCommand,PreviewSQLTableCommand,)+from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEnginefrom marimo._sql.engines.types import EngineCatalogfrom marimo._sql.get_engines import engine_to_data_source_connectionfrom marimo._tracer import kernel_tracer@@ -75,6 +76,8 @@ class DatasetCallbacks:"""Get engines that support catalog operations.Returns an error if the connection does not support catalog operations."""variable_name = cast(VariableName, variable_name)+ if variable_name == INTERNAL_DUCKDB_ENGINE:+ return DuckDBEngine(), Noneconnection, error = self._kernel.get_sql_connection(variable_name)if error is not None or connection is None:return None, errordiff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.pyindex 66150a9a93..b0e8f8d1ad 100644--- a/marimo/_sql/engines/duckdb.py+++ b/marimo/_sql/engines/duckdb.py@@ -2,6 +2,7 @@from __future__ import annotationsfrom contextlib import contextmanager, nullcontext+from dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Literal, Optional, castfrom marimo import _loggers@@ -20,7 +21,7 @@ from marimo._runtime.context.types import ()from marimo._sql.engines.types import InferenceConfig, SQLConnectionfrom marimo._sql.sql_quoting import quote_qualified_name-from marimo._sql.utils import convert_to_output, wrapped_sql+from marimo._sql.utils import convert_to_output, is_duckdb_v2, wrapped_sqlfrom marimo._types.ids import VariableNameLOGGER = _loggers.marimo_logger()@@ -35,6 +36,13 @@ if TYPE_CHECKING:INTERNAL_DUCKDB_ENGINE = cast(VariableName, "__marimo_duckdb")+@dataclass(frozen=True)+class _CatalogSchema:+ database: str+ oid: int+ path: tuple[str, ...]++class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):"""DuckDB SQL engine."""@@ -195,32 +203,28 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):for (name,) in rows}if schemas_resolved and databases:- # duckdb_schemas() scans every catalog before applying filters.- # Query it once so remote namespaces are not fetched per database.- schema_rows = self._query_catalog(- "SELECT database_name, schema_name FROM duckdb_schemas() "- "WHERE schema_name NOT IN ('information_schema', 'pg_catalog') "- "ORDER BY database_name, schema_name"- )- for database_name, schema_name in schema_rows:- if database_name in databases:- databases[database_name].schemas.append(- self._make_schema(- schema_name,- database_name,- include_tables=tables_resolved,- include_table_details=False,+ schemas = self._get_catalog_schemas()+ for database in databases.values():+ database.schemas = self._make_schemas(+ [+ schema+ for schema in schemas+ if schema.database == database.name+ ],+ include_tables=tables_resolved,+ include_table_details=False,+ )+ if details_resolved:+ self._load_table_details(+ {+ (database.name, path, table.name): table+ for database in databases.values()+ for path, table in self._schema_tables(+ database.schemas)- )- if details_resolved:- self._load_table_details(- {- (database.name, schema.name, table.name): table- for database in databases.values()- for schema in database.schemas- for table in schema.tables- }- )+ },+ schemas=schemas,+ )return list(databases.values())def _query_catalog(@@ -238,6 +242,35 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):LOGGER.debug("Skipping closed DuckDB connection")return []+ def _get_catalog_schemas(+ self, database: str | None = None+ ) -> list[_CatalogSchema]:+ parent_column = "parent_schema_oid" if is_duckdb_v2() else "NULL"+ query = (+ "SELECT database_name, oid, schema_name, "+ f"{parent_column} FROM duckdb_schemas() "+ "WHERE NOT (internal AND schema_name IN ('information_schema', 'pg_catalog'))"+ )+ params = None+ if database is not None:+ query += " AND database_name = ?"+ params = [database]+ rows = self._query_catalog(+ query + " ORDER BY database_name, schema_name", params+ )+ names = {(db, oid): (name, parent) for db, oid, name, parent in rows}++ def schema_path(database: str, oid: int) -> tuple[str, ...]:+ name, parent = names[database, oid]+ if parent is None:+ return (name,)+ return (*schema_path(database, parent), name)++ return [+ _CatalogSchema(db, oid, schema_path(db, oid))+ for db, oid, _, _ in rows+ ]+def get_schemas(self,*,@@ -246,46 +279,126 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):include_table_details: bool,schema_path: list[str] | None = None,) -> list[Schema]:- """List schemas without enumerating tables unless requested."""- if schema_path:- return []- if database is None:- database = self.get_default_database()+ """List immediate schemas, preserving deferred table discovery."""+ database = (+ database if database is not None else self.get_default_database()+ )if database is None:return []- rows = self._query_catalog(- "SELECT schema_name FROM duckdb_schemas() "- "WHERE database_name = ? "- "AND schema_name NOT IN ('information_schema', 'pg_catalog') "- "ORDER BY schema_name",- [database],+ schemas = self._get_catalog_schemas(database)+ result = self._make_schemas(+ schemas,+ parent_path=tuple(schema_path or ()),+ include_tables=include_tables,+ include_table_details=False,)- return [- self._make_schema(- name,- database,- include_tables=include_tables,- include_table_details=include_table_details,+ if include_tables and include_table_details:+ self._load_table_details(+ {+ (database, path, table.name): table+ for path, table in self._schema_tables(+ result, tuple(schema_path or ())+ )+ },+ schemas=schemas,+ database=database,+ schema_oids={+ schema.oid+ for schema in schemas+ if schema.path[: len(schema_path or ())]+ == tuple(schema_path or ())+ },)- for (name,) in rows- ]+ return result- def _make_schema(+ def _make_schemas(self,- name: str,- database: str,+ schemas: list[_CatalogSchema],*,+ parent_path: tuple[str, ...] = (),include_tables: bool,include_table_details: bool,- ) -> Schema:- tables = []- if include_tables:- tables = self.get_tables_in_schema(- schema=name,- database=database,- include_table_details=include_table_details,+ ) -> list[Schema]:+ children: dict[tuple[str, ...], list[_CatalogSchema]] = {}+ for schema in schemas:+ children.setdefault(schema.path[:-1], []).append(schema)++ def build(path: tuple[str, ...]) -> list[Schema]:+ return [+ Schema(+ name=schema.path[-1],+ tables=self._get_schema_tables(+ schema, include_table_details=include_table_details+ )+ if include_tables+ else [],+ tables_resolved=include_tables,+ child_schemas=build(schema.path) if include_tables else [],+ child_schemas_resolved=include_tables+ or schema.path not in children,+ )+ for schema in children.get(path, [])+ ]++ return build(parent_path)++ @staticmethod+ def _schema_tables(+ schemas: list[Schema], parent_path: tuple[str, ...] = ()+ ) -> Iterator[tuple[tuple[str, ...], DataTable]]:+ for schema in schemas:+ path = (*parent_path, schema.name)+ for table in schema.tables:+ yield path, table+ yield from DuckDBEngine._schema_tables(schema.child_schemas, path)++ def _find_schema(+ self, database: str, path: tuple[str, ...]+ ) -> _CatalogSchema | None:+ normalized_path = tuple(part.lower() for part in path)+ return next(+ (+ schema+ for schema in self._get_catalog_schemas(database)+ if tuple(part.lower() for part in schema.path)+ == normalized_path+ ),+ None,+ )++ def _table_rows(+ self, schema: _CatalogSchema, table_name: str | None = None+ ) -> list[tuple[str, DataTableType]]:+ predicate = "NOT internal AND database_name = ? AND schema_oid = ?"+ params: list[str | int] = [schema.database, schema.oid]+ if table_name is not None:+ predicate += " AND table_name = ?"+ params.append(table_name)+ return self._query_catalog(+ f"SELECT table_name, 'table' FROM duckdb_tables() WHERE {predicate} "+ f"UNION ALL SELECT view_name, 'view' FROM duckdb_views() WHERE "+ + predicate.replace("table_name", "view_name")+ + " ORDER BY 1",+ params * 2,+ )++ def _get_schema_tables(+ self, schema: _CatalogSchema, *, include_table_details: bool+ ) -> list[DataTable]:+ tables = {+ (schema.database, schema.path, name): self._make_table(+ name, schema.database, table_type=kind+ )+ for name, kind in self._table_rows(schema)+ }+ if include_table_details:+ self._load_table_details(+ tables,+ schemas=[schema],+ database=schema.database,+ schema_oids={schema.oid},)- return Schema(name=name, tables=tables, tables_resolved=include_tables)+ return list(tables.values())def get_tables_in_schema(self,@@ -295,56 +408,58 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):include_table_details: bool,schema_path: list[str] | None = None,) -> list[DataTable]:- """List table and view names without eagerly loading their columns."""- del schema_path- rows = self._query_catalog(- "SELECT table_name, table_type FROM information_schema.tables "- "WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",- [database, schema],+ catalog_schema = self._find_schema(+ database, tuple(schema_path or [schema])+ )+ if catalog_schema is None:+ return []+ return self._get_schema_tables(+ catalog_schema, include_table_details=include_table_details)- tables = {- (database, schema, name): self._make_table(- name,- database,- table_type="view" if kind == "VIEW" else "table",- )- for name, kind in rows- }- if include_table_details:- self._load_table_details(tables, database=database, schema=schema)- return list(tables.values())def _load_table_details(self,- tables: dict[tuple[str, str, str], DataTable],+ tables: dict[tuple[str, tuple[str, ...], str], DataTable],*,+ schemas: list[_CatalogSchema],database: str | None = None,- schema: str | None = None,+ schema_oids: set[int] | None = None,) -> None:if not tables:return+ paths = {+ (schema.database, schema.oid): schema.path for schema in schemas+ }query = (- "SELECT database_name, schema_name, table_name, column_name, data_type "+ "SELECT database_name, schema_oid, table_name, column_name, data_type ""FROM duckdb_columns() WHERE NOT internal")- params: list[str] = []+ params: list[str | int] = []if database is not None:query += " AND database_name = ?"params.append(database)- if schema is not None:- query += " AND schema_name = ?"- params.append(schema)+ if schema_oids is not None:+ if not schema_oids:+ return+ query += (+ " AND schema_oid IN ("+ + ", ".join("?" for _ in schema_oids)+ + ")"+ )+ params.extend(sorted(schema_oids))query += (- " ORDER BY database_name, schema_name, table_name, column_index"+ " ORDER BY database_name, schema_oid, table_name, column_index")try:rows = self._query_catalog(query, params)except Exception:- # A metadata failure must not hide tables already enumerated.LOGGER.warning("Failed to get DuckDB columns", exc_info=True)return- for db_name, schema_name, table_name, column_name, dtype in rows:- table = tables.get((db_name, schema_name, table_name))+ for db_name, oid, table_name, column_name, dtype in rows:+ path = paths.get((db_name, oid))+ if path is None:+ continue+ table = tables.get((db_name, path, table_name))if table is not None:table.columns.append(DataTableColumn(@@ -354,11 +469,11 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):sample_values=[],))- for (_, schema_name, _), table in tables.items():+ for (_, path, _), table in tables.items():# Some Iceberg catalogs expose a placeholder instead of columns.if len(table.columns) == 1 and table.columns[0].name == "__":table.columns = []- self._describe_table(table, schema_name)+ self._describe_table(table, path)if table.columns:table.num_columns = len(table.columns)@@ -388,29 +503,30 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):schema_path: list[str] | None = None,) -> DataTable | None:"""Describe only the requested table, including remote catalog tables."""- del schema_path- rows = self._query_catalog(- "SELECT table_type FROM information_schema.tables "- "WHERE table_catalog = ? AND table_schema = ? AND table_name = ?",- [database_name, schema_name, table_name],- )+ path = tuple(schema_path or [schema_name])+ schema = self._find_schema(database_name, path)+ if schema is None:+ return None+ rows = self._table_rows(schema, table_name)if not rows:return Nonetable = self._make_table(- table_name,- database_name,- table_type="view" if rows[0][0] == "VIEW" else "table",+ table_name, database_name, table_type=rows[0][1])- self._describe_table(table, schema_name)+ self._describe_table(table, path)return table if table.columns else None- def _describe_table(self, table: DataTable, schema: str) -> None:+ def _describe_table(+ self, table: DataTable, schema_path: tuple[str, ...]+ ) -> None:import duckdbconnection = cast(duckdb.DuckDBPyConnection, self._connection or duckdb)- qualified_name = quote_qualified_name(table.source, schema, table.name)+ qualified_name = quote_qualified_name(+ table.source, *schema_path, table.name+ )with self._install_connection(connection):table.columns = get_table_columns(connection, qualified_name)if table.columns:diff --git a/tests/_data/test_sql_summaries.py b/tests/_data/test_sql_summaries.pyindex 8397875dd4..f74e789e70 100644--- a/tests/_data/test_sql_summaries.py+++ b/tests/_data/test_sql_summaries.py@@ -160,3 +160,56 @@ def test_geometry_column_type_and_stats():assert summary.unique is Nonefinally:duckdb.execute("DROP TABLE geo_stats_table")++[email protected](not HAS_DEPS, reason="optional dependencies not installed")+def test_nested_column_type() -> None:+ import duckdb++ from marimo._sql.utils import is_duckdb_v2++ if not is_duckdb_v2():+ pytest.skip("Nested schemas require DuckDB 2")+ try:+ duckdb.execute(+ "CREATE SCHEMA type_test; CREATE SCHEMA type_test.reports; "+ "CREATE TABLE type_test.reports.sales (amount INT); "+ 'CREATE SCHEMA "type_test.reports"; '+ 'CREATE TABLE "type_test.reports".sales (amount VARCHAR)'+ )+ assert (+ get_column_type('"memory"."type_test"."reports"."sales"', "amount")+ == "integer"+ )+ assert (+ get_column_type('"memory"."type_test.reports"."sales"', "amount")+ == "string"+ )+ finally:+ duckdb.execute(+ 'DROP SCHEMA IF EXISTS type_test CASCADE; DROP SCHEMA IF EXISTS "type_test.reports" CASCADE'+ )++[email protected](not HAS_DEPS, reason="optional dependencies not installed")+ ("schema", "table"),+ [('column_type_"schema', "t.x"), ("column_type_schema", 't"x')],+)+def test_column_type_with_quoted_identifiers(schema: str, table: str) -> None:+ import duckdb++ from marimo._sql.sql_quoting import (+ quote_qualified_name,+ quote_sql_identifier,+ )++ qualified = quote_qualified_name("memory", schema, table)+ try:+ duckdb.execute(f"CREATE SCHEMA {quote_sql_identifier(schema)}")+ duckdb.execute(f'CREATE TABLE {qualified} ("c""x" INT)')+ assert get_column_type(qualified, 'c"x') == "integer"+ finally:+ duckdb.execute(+ f"DROP SCHEMA IF EXISTS memory.{quote_sql_identifier(schema)} CASCADE"+ )diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.pyindex 82e74991a2..aa609bd3f7 100644--- a/tests/_runtime/test_runtime_datasets.py+++ b/tests/_runtime/test_runtime_datasets.py@@ -84,6 +84,43 @@ async def connection_requests() -> list[ExecuteCellCommand]:@pytest.mark.skipif(not HAS_SQL, reason="SQL deps not available")class TestPreviewSQLTable:+ async def test_internal_duckdb_nested_table(+ self, mocked_kernel: MockedKernel+ ) -> None:+ import duckdb++ from marimo._sql.utils import is_duckdb_v2++ if not is_duckdb_v2():+ pytest.skip("Nested schemas require DuckDB 2")+ try:+ duckdb.execute(+ "CREATE SCHEMA preview_nested; CREATE SCHEMA preview_nested.reports; CREATE TABLE preview_nested.reports.sales (amount INT)"+ )+ await mocked_kernel.k.handle_message(+ PreviewSQLTableCommand(+ request_id=RequestId("nested"),+ engine=INTERNAL_DUCKDB_ENGINE,+ database="memory",+ schema="reports",+ schema_path=["preview_nested", "reports"],+ table_name="sales",+ )+ )+ results = [+ op+ for op in mocked_kernel.stream.operations+ if isinstance(op, SQLTablePreviewNotification)+ ]+ assert len(results) == 1+ assert results[0].error is None+ assert results[0].table is not None+ assert results[0].table.columns[0].name == "amount"+ finally:+ duckdb.execute(+ "DROP SCHEMA IF EXISTS memory.preview_nested CASCADE"+ )+async def test_non_existent_engine(self, mocked_kernel: MockedKernel) -> None:diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.pyindex 17886d6342..70d25249a3 100644--- a/tests/_sql/test_duckdb.py+++ b/tests/_sql/test_duckdb.py@@ -511,8 +511,7 @@ def test_duckdb_discovery_depth(schemas_enabled)assert (- any("information_schema.tables" in sql for sql in queries)- == tables_enabled+ any("duckdb_tables()" in sql for sql in queries) == tables_enabled)assert sum("duckdb_columns()" in sql for sql in queries) == int(details_enabled@@ -813,7 +812,10 @@ def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:"Column metadata unavailable")if failure == "placeholder":- return [("memory", "main", "example", "__", "INTEGER")]+ oid = connection.execute(+ "SELECT oid FROM duckdb_schemas() WHERE database_name = 'memory' AND schema_name = 'main'"+ ).fetchone()[0]+ return [("memory", oid, "example", "__", "INTEGER")]return []return query(sql, params)@@ -840,3 +842,90 @@ def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:)]assert describe.call_count == int(failure == "placeholder")++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")[email protected]("include_tables", [False, True])+def test_duckdb_nested_schema_discovery(include_tables: bool) -> None:+ import duckdb++ from marimo._sql.utils import is_duckdb_v2++ if not is_duckdb_v2():+ pytest.skip("Nested schemas require DuckDB 2")+ with duckdb.connect() as connection:+ for parent in ("finance", "other", "finance.reports"):+ connection.execute(f"CREATE SCHEMA {quote_qualified_name(parent)}")+ for parent, dtype in (("finance", "INTEGER"), ("other", "VARCHAR")):+ connection.execute(+ f"CREATE SCHEMA {quote_qualified_name(parent, 'reports')}"+ )+ connection.execute(+ f"CREATE TABLE {quote_qualified_name(parent, 'reports', 'sales')} (amount {dtype})"+ )+ connection.execute(+ f"CREATE VIEW {quote_qualified_name(parent, 'reports', 'summary')} AS SELECT * FROM {quote_qualified_name(parent, 'reports', 'sales')}"+ )+ connection.execute(+ 'CREATE TABLE "finance.reports".sales (literal BOOLEAN)'+ )+ engine = DuckDBEngine(connection)+ databases = engine.get_databases(+ include_schemas=True,+ include_tables=include_tables,+ include_table_details=True,+ )+ schemas = {+ s.name: s+ for db in databases+ if db.name == "memory"+ for s in db.schemas+ }+ assert set(schemas) == {"finance", "other", "finance.reports", "main"}+ assert schemas["finance"].child_schemas_resolved == include_tables+ assert schemas["finance"].tables == []+ for parent, dtype in (("finance", "INTEGER"), ("other", "VARCHAR")):+ children = (+ schemas[parent].child_schemas+ if include_tables+ else engine.get_schemas(+ database="memory",+ schema_path=[parent],+ include_tables=True,+ include_table_details=True,+ )+ )+ assert len(children) == 1+ assert children[0].name == "reports"+ assert [(t.name, t.type) for t in children[0].tables] == [+ ("sales", "table"),+ ("summary", "view"),+ ]+ assert all(+ t.columns[0].external_type == dtype for t in children[0].tables+ )+ for name in ("sales", "summary"):+ details = engine.get_table_details(+ database_name="memory",+ schema_name="reports",+ schema_path=[parent, "reports"],+ table_name=name,+ )+ assert details is not None+ assert details.columns[0].external_type == dtype+ literal = engine.get_tables_in_schema(+ database="memory",+ schema="finance.reports",+ schema_path=["finance.reports"],+ include_table_details=True,+ )+ assert literal[0].columns[0].name == "literal"+ assert (+ engine.get_tables_in_schema(+ database="memory",+ schema="reports",+ schema_path=["missing", "reports"],+ include_table_details=True,+ )+ == []+ )