Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(sql): preserve nested schema paths in datasource browsing (#11458)

Shahmir Varqha0423673a73e9
diff --git a/frontend/src/components/datasources/__tests__/column-preview.test.tsx b/frontend/src/components/datasources/__tests__/column-preview.test.tsx
index c6f0784339..0202d328f6 100644
--- a/frontend/src/components/datasources/__tests__/column-preview.test.tsx
+++ b/frontend/src/components/datasources/__tests__/column-preview.test.tsx
@@ -79,7 +79,9 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {
it("qualifies with database and schema for flat engines", () => {
const client = renderPreview(ctx({ database: "db", schema: "public" }));
expect(client.previewDatasetColumn).toHaveBeenCalledWith(
- expect.objectContaining({ fullyQualifiedTableName: "db.public.users" }),
+ expect.objectContaining({
+ fullyQualifiedTableName: '"db"."public"."users"',
+ }),
);
});
@@ -87,7 +89,7 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {
// Regression: previously produced "db..users".
const client = renderPreview(ctx({ database: "db", schema: "" }));
expect(client.previewDatasetColumn).toHaveBeenCalledWith(
- expect.objectContaining({ fullyQualifiedTableName: "db.users" }),
+ expect.objectContaining({ fullyQualifiedTableName: '"db"."users"' }),
);
});
@@ -98,7 +100,37 @@ describe("DatasetColumnPreview fullyQualifiedTableName", () => {
);
expect(client.previewDatasetColumn).toHaveBeenCalledWith(
expect.objectContaining({
- fullyQualifiedTableName: "top.nested.deep.users",
+ fullyQualifiedTableName: '"top"."nested"."deep"."users"',
+ }),
+ );
+ });
+
+ it("quotes a literal dot as one schema segment", () => {
+ const client = renderPreview(
+ ctx({
+ database: "memory",
+ schema: "finance.reports",
+ schemaPath: ["finance.reports"],
+ }),
+ );
+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(
+ expect.objectContaining({
+ fullyQualifiedTableName: '"memory"."finance.reports"."users"',
+ }),
+ );
+ });
+
+ it("escapes quotes inside schema identifiers", () => {
+ const client = renderPreview(
+ ctx({
+ database: "memory",
+ schema: 'fi"nance',
+ schemaPath: ['fi"nance', "reports"],
+ }),
+ );
+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(
+ expect.objectContaining({
+ fullyQualifiedTableName: '"memory"."fi""nance"."reports"."users"',
}),
);
});
diff --git a/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsx b/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsx
new file mode 100644
index 0000000000..3032406430
--- /dev/null
+++ b/frontend/src/components/datasources/__tests__/datasources-interactions.test.tsx
@@ -0,0 +1,181 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import {
+ act,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from "@testing-library/react";
+import { createStore, Provider } from "jotai";
+import { beforeAll, describe, expect, it, vi } from "vitest";
+import { MockRequestClient } from "@/__mocks__/requests";
+import { Command, CommandList } from "@/components/ui/command";
+import { TooltipProvider } from "@/components/ui/tooltip";
+import {
+ dataSourceConnectionsAtom,
+ type SQLTableContext,
+} from "@/core/datasets/data-source-connections";
+import { DUCKDB_ENGINE } from "@/core/datasets/engines";
+import { datasetsAtom, exportedForTesting } from "@/core/datasets/state";
+import type { DataTable } from "@/core/kernel/messages";
+import { requestClientAtom } from "@/core/network/requests";
+import {
+ DataSources,
+ hideEmptyDatasourcesAtom,
+ TableList,
+} from "../datasources";
+
+beforeAll(() => {
+ HTMLElement.prototype.scrollIntoView = vi.fn();
+});
+
+const table: DataTable = {
+ name: "reports",
+ columns: [
+ {
+ name: "amount",
+ type: "integer",
+ external_type: "INTEGER",
+ sample_values: [],
+ },
+ ],
+ source: "memory",
+ source_type: "duckdb",
+ type: "table",
+ engine: null,
+ indexes: null,
+ num_columns: 1,
+ num_rows: null,
+ variable_name: null,
+ primary_keys: null,
+};
+
+describe("Datasource column preview responses", () => {
+ it.each([
+ { path: ["main"], qualifiedName: '"memory"."main"."reports"' },
+ {
+ path: ["finance", "reports"],
+ qualifiedName: '"memory"."finance"."reports"."reports"',
+ },
+ {
+ path: ["finance.reports"],
+ qualifiedName: '"memory"."finance.reports"."reports"',
+ },
+ {
+ path: ['fi"nance', "reports"],
+ qualifiedName: '"memory"."fi""nance"."reports"."reports"',
+ },
+ ])(
+ "renders the returned preview for $path",
+ async ({ path, qualifiedName }) => {
+ const store = createStore();
+ const client = MockRequestClient.create();
+ store.set(requestClientAtom, client);
+ const context: SQLTableContext = {
+ engine: DUCKDB_ENGINE,
+ database: "memory",
+ schema: path.at(-1) ?? "",
+ schemaPath: path,
+ dialect: "duckdb",
+ };
+ render(
+ <Provider store={store}>
+ <TooltipProvider>
+ <Command shouldFilter={false}>
+ <CommandList>
+ <TableList tables={[table]} sqlTableContext={context} />
+ </CommandList>
+ </Command>
+ </TooltipProvider>
+ </Provider>,
+ );
+
+ fireEvent.click(await screen.findByText("reports"));
+ fireEvent.click(await screen.findByText("amount"));
+ await waitFor(() => {
+ expect(client.previewDatasetColumn).toHaveBeenCalledWith(
+ expect.objectContaining({ fullyQualifiedTableName: qualifiedName }),
+ );
+ });
+ expect(screen.getByText("Loading...")).toBeInTheDocument();
+
+ const actions = exportedForTesting.createActions((action) => {
+ store.set(datasetsAtom, (state) =>
+ exportedForTesting.reducer(state, action),
+ );
+ });
+ act(() => {
+ actions.addColumnPreview({
+ table_name: qualifiedName,
+ column_name: "amount",
+ stats: { min: 42 },
+ error: null,
+ missing_packages: null,
+ chart_spec: null,
+ chart_code: null,
+ });
+ });
+
+ expect(await screen.findByText("42")).toBeInTheDocument();
+ expect(screen.queryByText("Loading...")).not.toBeInTheDocument();
+ },
+ );
+});
+
+it("activates a schema and a table with the same path independently using the keyboard", async () => {
+ const store = createStore();
+ store.set(requestClientAtom, MockRequestClient.create());
+ store.set(hideEmptyDatasourcesAtom, false);
+ store.set(dataSourceConnectionsAtom, {
+ latestEngineSelected: DUCKDB_ENGINE,
+ connectionsMap: new Map([
+ [
+ DUCKDB_ENGINE,
+ {
+ name: DUCKDB_ENGINE,
+ dialect: "duckdb",
+ source: "duckdb",
+ display_name: "DuckDB (In-Memory)",
+ databases: [
+ {
+ name: "memory",
+ dialect: "duckdb",
+ schemas: [
+ {
+ name: "finance",
+ tables: [table],
+ child_schemas: [{ name: "reports", tables: [] }],
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ ]),
+ });
+ render(
+ <Provider store={store}>
+ <TooltipProvider>
+ <DataSources />
+ </TooltipProvider>
+ </Provider>,
+ );
+ fireEvent.click(await screen.findByText("memory"));
+ fireEvent.click(await screen.findByText("finance"));
+ await waitFor(() => {
+ expect(screen.getAllByText("reports")).toHaveLength(2);
+ });
+
+ const input = screen.getByRole("combobox");
+ input.focus();
+ fireEvent.keyDown(input, { key: "ArrowDown" });
+ fireEvent.keyDown(input, { key: "Enter" });
+ expect(await screen.findByText("No tables found")).toBeInTheDocument();
+ expect(screen.queryByText("amount")).not.toBeInTheDocument();
+
+ fireEvent.keyDown(input, { key: "ArrowDown" });
+ fireEvent.keyDown(input, { key: "Enter" });
+ expect(await screen.findByText("amount")).toBeInTheDocument();
+ expect(screen.getByText("No tables found")).toBeInTheDocument();
+});
diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.ts
index bad765cf5b..f2d2957688 100644
--- a/frontend/src/components/datasources/__tests__/utils.test.ts
+++ b/frontend/src/components/datasources/__tests__/utils.test.ts
@@ -34,6 +34,37 @@ describe("sqlCode", () => {
name: "email" as const,
} as DataTableColumn;
+ it.each([
+ {
+ path: ["finance", "reports"],
+ expected: '"memory"."finance"."reports"."users"',
+ },
+ { path: ["finance.reports"], expected: '"finance.reports"."users"' },
+ {
+ path: ['fi"nance', "reports"],
+ expected: '"memory"."fi""nance"."reports"."users"',
+ },
+ {
+ path: ["main", "reports"],
+ expected: '"memory"."main"."reports"."users"',
+ },
+ ])("preserves schema path $path in generated SQL", ({ path, expected }) => {
+ const result = sqlCode({
+ table: mockTable,
+ columnName: "*",
+ sqlTableContext: {
+ engine: DUCKDB_ENGINE,
+ dialect: "duckdb",
+ database: "memory",
+ defaultDatabase: "memory",
+ defaultSchema: "main",
+ schema: path.at(-1)!,
+ schemaPath: path,
+ },
+ });
+ expect(result).toContain(`FROM ${expected}`);
+ });
+
describe("basic SQL generation", () => {
it("should generate basic SQL without sqlTableContext", () => {
const result = sqlCode({ table: mockTable, columnName: mockColumn.name });
@@ -484,7 +515,7 @@ describe("tableUniqueId", () => {
it("uses database + schema for flat engines", () => {
expect(tableUniqueId(ctx({ database: "db", schema: "public" }), "t")).toBe(
- "db.public.t",
+ '["db","public","t"]',
);
});
@@ -495,7 +526,7 @@ describe("tableUniqueId", () => {
ctx({ database: "top", schema: "nested", schemaPath: ["nested"] }),
"t",
),
- ).toBe("top.nested.t");
+ ).toBe('["top","nested","t"]');
});
it("includes the full schema path for deeply nested namespaces", () => {
@@ -508,7 +539,7 @@ describe("tableUniqueId", () => {
}),
"t",
),
- ).toBe("top.nested.deep.t");
+ ).toBe('["top","nested","deep","t"]');
});
it("falls back to the flat schema when schemaPath is empty", () => {
@@ -517,16 +548,36 @@ describe("tableUniqueId", () => {
ctx({ database: "db", schema: "public", schemaPath: [] }),
"t",
),
- ).toBe("db.public.t");
+ ).toBe('["db","public","t"]');
});
it("does not emit a double dot for schemaless tables", () => {
expect(
tableUniqueId(ctx({ database: "db", schema: "", schemaPath: [] }), "t"),
- ).toBe("db.t");
+ ).toBe('["db","t"]');
expect(tableUniqueId(ctx({ database: "db", schema: "" }), "t")).toBe(
- "db.t",
+ '["db","t"]',
+ );
+ });
+
+ it("keeps literal-dot and nested schemas distinct", () => {
+ const nested = tableUniqueId(
+ ctx({
+ database: "memory",
+ schema: "reports",
+ schemaPath: ["finance", "reports"],
+ }),
+ "sales",
+ );
+ const literal = tableUniqueId(
+ ctx({
+ database: "memory",
+ schema: "finance.reports",
+ schemaPath: ["finance.reports"],
+ }),
+ "sales",
);
+ expect(nested).not.toBe(literal);
});
});
diff --git a/frontend/src/components/datasources/column-preview.tsx b/frontend/src/components/datasources/column-preview.tsx
index d68fd24302..ce85747d42 100644
--- a/frontend/src/components/datasources/column-preview.tsx
+++ b/frontend/src/components/datasources/column-preview.tsx
@@ -29,7 +29,7 @@ import { Button } from "../ui/button";
import { Tooltip } from "../ui/tooltip";
import { ColumnPreviewContainer } from "./components";
import { InstallPackageButton } from "./install-package-button";
-import { convertStatsName, sqlCode, tableUniqueId } from "./utils";
+import { convertStatsName, fullyQualifiedTableName, sqlCode } from "./utils";
export const DatasetColumnPreview: React.FC<{
table: DataTable;
@@ -48,7 +48,10 @@ export const DatasetColumnPreview: React.FC<{
tableName: table.name,
columnName: column.name,
sourceType: table.source_type,
- fullyQualifiedTableName: tableUniqueId(sqlTableContext, table.name),
+ fullyQualifiedTableName: fullyQualifiedTableName(
+ sqlTableContext,
+ table.name,
+ ),
});
};
diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsx
index c18af29e64..b5060169ed 100644
--- a/frontend/src/components/datasources/datasources.tsx
+++ b/frontend/src/components/datasources/datasources.tsx
@@ -78,6 +78,7 @@ import {
areChildSchemasResolved,
areSchemasResolved,
areTablesResolved,
+ fullyQualifiedTableName,
isSchemaless,
schemaSubtreeMatchesSearch,
shouldExpandDatabaseForSearch,
@@ -637,7 +638,7 @@ const SchemaNode: React.FC<SchemaNodeProps> = (props) => {
const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);
const [prevExpandForSearch, setPrevExpandForSearch] =
React.useState(expandForSearch);
- const uniqueValue = `${databaseName}:${schemaPath.join(".")}`;
+ const uniqueValue = JSON.stringify(["schema", databaseName, ...schemaPath]);
const childSchemas = schema.child_schemas ?? [];
// Re-sync when the query changes or when newly resolved data turns this
@@ -1095,9 +1096,7 @@ const DatasetColumnItem: React.FC<{
column={column}
onAddColumnChart={handleAddColumn}
preview={columnsPreviews.get(
- sqlTableContext
- ? `${sqlTableContext.database}.${sqlTableContext.schema}.${table.name}:${column.name}`
- : `${table.name}:${column.name}`,
+ `${fullyQualifiedTableName(sqlTableContext, table.name)}:${column.name}`,
)}
sqlTableContext={sqlTableContext}
/>
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.ts
index 904d152acb..4a9d5906ec 100644
--- a/frontend/src/components/datasources/utils.ts
+++ b/frontend/src/components/datasources/utils.ts
@@ -13,26 +13,29 @@ import type {
import { logNever } from "@/utils/assertNever";
import type { ColumnHeaderStatsKey } from "../data-table/types";
-/**
- * Stable id for a table node in the datasources tree.
- *
- * schemaPath already includes the leaf schema for nested namespaces, so use it
- * when present and fall back to the flat schema name otherwise (avoids
- * duplicating the leaf, e.g. `top.nested.nested.table`).
- */
+function tablePath(context: SQLTableContext, tableName: string): string[] {
+ const segments = context.schemaPath?.length
+ ? context.schemaPath
+ : [context.schema];
+ return [context.database, ...segments.filter(Boolean), tableName];
+}
+
export function tableUniqueId(
- sqlTableContext: SQLTableContext | undefined,
+ context: SQLTableContext | undefined,
tableName: string,
): string {
- if (!sqlTableContext) {
- return tableName;
- }
- const segments = (
- sqlTableContext.schemaPath?.length
- ? sqlTableContext.schemaPath
- : [sqlTableContext.schema]
- ).filter(Boolean);
- return [sqlTableContext.database, ...segments, tableName].join(".");
+ return context ? JSON.stringify(tablePath(context, tableName)) : tableName;
+}
+
+export function fullyQualifiedTableName(
+ context: SQLTableContext | undefined,
+ tableName: string,
+): string {
+ return context
+ ? getFormatter(context.dialect).formatTablePath(
+ tablePath(context, tableName),
+ )
+ : tableName;
}
// Some databases have no schemas, so we don't show it (eg. Clickhouse)
@@ -114,6 +117,10 @@ const defaultFormatter: SqlCodeFormatter = {
`SELECT ${columnName} FROM ${tableName} LIMIT 100`,
};
+function quoteIdentifier(identifier: string): string {
+ return `"${identifier.replaceAll('"', '""')}"`;
+}
+
function getFormatter(dialect: string): SqlCodeFormatter {
dialect = dialect.toLowerCase();
if (!isKnownDialect(dialect)) {
@@ -146,9 +153,9 @@ function getFormatter(dialect: string): SqlCodeFormatter {
// Quote column and table names to avoid raising errors on weird characters
return {
formatTablePath: (tablePath: string[]) =>
- tablePath.map((part) => `"${part}"`).join("."),
+ tablePath.map(quoteIdentifier).join("."),
formatSelectClause: (columnName: string, tableName: string) =>
- `SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,
+ `SELECT ${columnName === "*" ? "*" : quoteIdentifier(columnName)} FROM ${tableName} LIMIT 100`,
};
case "db2":
case "db2i":
@@ -194,6 +201,7 @@ export function sqlCode({
engine,
schema,
defaultSchema,
+ schemaPath,
defaultDatabase,
database,
dialect,
@@ -207,12 +215,13 @@ export function sqlCode({
}
} else {
// Include schema if it's not the default schema
- if (schema !== defaultSchema) {
- tablePath.unshift(schema);
+ const path = schemaPath?.length ? schemaPath : [schema];
+ if (path.length > 1 || path[0] !== defaultSchema) {
+ tablePath.unshift(...path);
}
- // Include database if it's not the default database
- if (database !== defaultDatabase) {
+ // Nested schema names can also be attached catalog names.
+ if (path.length > 1 || database !== defaultDatabase) {
tablePath.unshift(database);
}
}
diff --git a/marimo/_data/sql_summaries.py b/marimo/_data/sql_summaries.py
index b7c7f63fab..92d891a06e 100644
--- a/marimo/_data/sql_summaries.py
+++ b/marimo/_data/sql_summaries.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from marimo._data.get_datasets import _db_type_to_data_type
from marimo._data.models import ColumnStats, DataType
from marimo._sql.sql_quoting import parse_fully_qualified_table_name
-from marimo._sql.utils import execute_duckdb_sql, wrapped_sql
+from marimo._sql.utils import execute_duckdb_sql, is_duckdb_v2, wrapped_sql
def get_sql_stats(
@@ -146,8 +146,14 @@ def get_column_type(
Get the type of a column in a SQL table.
"""
- # First, get the column info and data type
- if "." in fully_qualified_table_name:
+ # DuckDB 1.0's pragma_table_info does not preserve escaped identifier quotes.
+ if is_duckdb_v2():
+ table_name = fully_qualified_table_name
+ column_info_result: tuple[str] | None = execute_duckdb_sql(
+ "SELECT type FROM pragma_table_info(?) WHERE name = ?",
+ [fully_qualified_table_name, column_name],
+ ).fetchone()
+ elif "." in fully_qualified_table_name:
# Fully qualified table name
db_name, schema_name, table_name = parse_fully_qualified_table_name(
fully_qualified_table_name
@@ -160,7 +166,7 @@ def get_column_type(
AND table_catalog = $3
AND column_name = $4
"""
- column_info_result: tuple[str] | None = execute_duckdb_sql(
+ column_info_result = execute_duckdb_sql(
column_info_query,
[table_name, schema_name, db_name, column_name],
).fetchone()
diff --git a/marimo/_runtime/callbacks/datasets.py b/marimo/_runtime/callbacks/datasets.py
index 4f32a96336..d9fd2d848b 100644
--- a/marimo/_runtime/callbacks/datasets.py
+++ b/marimo/_runtime/callbacks/datasets.py
@@ -29,6 +29,7 @@ from marimo._runtime.commands import (
PreviewDatasetColumnCommand,
PreviewSQLTableCommand,
)
+from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEngine
from marimo._sql.engines.types import EngineCatalog
from marimo._sql.get_engines import engine_to_data_source_connection
from marimo._tracer import kernel_tracer
@@ -75,6 +76,8 @@ class DatasetCallbacks:
"""Get engines that support catalog operations.
Returns an error if the connection does not support catalog operations."""
variable_name = cast(VariableName, variable_name)
+ if variable_name == INTERNAL_DUCKDB_ENGINE:
+ return DuckDBEngine(), None
connection, error = self._kernel.get_sql_connection(variable_name)
if error is not None or connection is None:
return None, error
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.py
index 66150a9a93..b0e8f8d1ad 100644
--- a/marimo/_sql/engines/duckdb.py
+++ b/marimo/_sql/engines/duckdb.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from contextlib import contextmanager, nullcontext
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
from marimo import _loggers
@@ -20,7 +21,7 @@ from marimo._runtime.context.types import (
)
from marimo._sql.engines.types import InferenceConfig, SQLConnection
from marimo._sql.sql_quoting import quote_qualified_name
-from marimo._sql.utils import convert_to_output, wrapped_sql
+from marimo._sql.utils import convert_to_output, is_duckdb_v2, wrapped_sql
from marimo._types.ids import VariableName
LOGGER = _loggers.marimo_logger()
@@ -35,6 +36,13 @@ if TYPE_CHECKING:
INTERNAL_DUCKDB_ENGINE = cast(VariableName, "__marimo_duckdb")
+@dataclass(frozen=True)
+class _CatalogSchema:
+ database: str
+ oid: int
+ path: tuple[str, ...]
+
+
class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
"""DuckDB SQL engine."""
@@ -195,32 +203,28 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
for (name,) in rows
}
if schemas_resolved and databases:
- # duckdb_schemas() scans every catalog before applying filters.
- # Query it once so remote namespaces are not fetched per database.
- schema_rows = self._query_catalog(
- "SELECT database_name, schema_name FROM duckdb_schemas() "
- "WHERE schema_name NOT IN ('information_schema', 'pg_catalog') "
- "ORDER BY database_name, schema_name"
- )
- for database_name, schema_name in schema_rows:
- if database_name in databases:
- databases[database_name].schemas.append(
- self._make_schema(
- schema_name,
- database_name,
- include_tables=tables_resolved,
- include_table_details=False,
+ schemas = self._get_catalog_schemas()
+ for database in databases.values():
+ database.schemas = self._make_schemas(
+ [
+ schema
+ for schema in schemas
+ if schema.database == database.name
+ ],
+ include_tables=tables_resolved,
+ include_table_details=False,
+ )
+ if details_resolved:
+ self._load_table_details(
+ {
+ (database.name, path, table.name): table
+ for database in databases.values()
+ for path, table in self._schema_tables(
+ database.schemas
)
- )
- if details_resolved:
- self._load_table_details(
- {
- (database.name, schema.name, table.name): table
- for database in databases.values()
- for schema in database.schemas
- for table in schema.tables
- }
- )
+ },
+ schemas=schemas,
+ )
return list(databases.values())
def _query_catalog(
@@ -238,6 +242,35 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
LOGGER.debug("Skipping closed DuckDB connection")
return []
+ def _get_catalog_schemas(
+ self, database: str | None = None
+ ) -> list[_CatalogSchema]:
+ parent_column = "parent_schema_oid" if is_duckdb_v2() else "NULL"
+ query = (
+ "SELECT database_name, oid, schema_name, "
+ f"{parent_column} FROM duckdb_schemas() "
+ "WHERE NOT (internal AND schema_name IN ('information_schema', 'pg_catalog'))"
+ )
+ params = None
+ if database is not None:
+ query += " AND database_name = ?"
+ params = [database]
+ rows = self._query_catalog(
+ query + " ORDER BY database_name, schema_name", params
+ )
+ names = {(db, oid): (name, parent) for db, oid, name, parent in rows}
+
+ def schema_path(database: str, oid: int) -> tuple[str, ...]:
+ name, parent = names[database, oid]
+ if parent is None:
+ return (name,)
+ return (*schema_path(database, parent), name)
+
+ return [
+ _CatalogSchema(db, oid, schema_path(db, oid))
+ for db, oid, _, _ in rows
+ ]
+
def get_schemas(
self,
*,
@@ -246,46 +279,126 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
include_table_details: bool,
schema_path: list[str] | None = None,
) -> list[Schema]:
- """List schemas without enumerating tables unless requested."""
- if schema_path:
- return []
- if database is None:
- database = self.get_default_database()
+ """List immediate schemas, preserving deferred table discovery."""
+ database = (
+ database if database is not None else self.get_default_database()
+ )
if database is None:
return []
- rows = self._query_catalog(
- "SELECT schema_name FROM duckdb_schemas() "
- "WHERE database_name = ? "
- "AND schema_name NOT IN ('information_schema', 'pg_catalog') "
- "ORDER BY schema_name",
- [database],
+ schemas = self._get_catalog_schemas(database)
+ result = self._make_schemas(
+ schemas,
+ parent_path=tuple(schema_path or ()),
+ include_tables=include_tables,
+ include_table_details=False,
)
- return [
- self._make_schema(
- name,
- database,
- include_tables=include_tables,
- include_table_details=include_table_details,
+ if include_tables and include_table_details:
+ self._load_table_details(
+ {
+ (database, path, table.name): table
+ for path, table in self._schema_tables(
+ result, tuple(schema_path or ())
+ )
+ },
+ schemas=schemas,
+ database=database,
+ schema_oids={
+ schema.oid
+ for schema in schemas
+ if schema.path[: len(schema_path or ())]
+ == tuple(schema_path or ())
+ },
)
- for (name,) in rows
- ]
+ return result
- def _make_schema(
+ def _make_schemas(
self,
- name: str,
- database: str,
+ schemas: list[_CatalogSchema],
*,
+ parent_path: tuple[str, ...] = (),
include_tables: bool,
include_table_details: bool,
- ) -> Schema:
- tables = []
- if include_tables:
- tables = self.get_tables_in_schema(
- schema=name,
- database=database,
- include_table_details=include_table_details,
+ ) -> list[Schema]:
+ children: dict[tuple[str, ...], list[_CatalogSchema]] = {}
+ for schema in schemas:
+ children.setdefault(schema.path[:-1], []).append(schema)
+
+ def build(path: tuple[str, ...]) -> list[Schema]:
+ return [
+ Schema(
+ name=schema.path[-1],
+ tables=self._get_schema_tables(
+ schema, include_table_details=include_table_details
+ )
+ if include_tables
+ else [],
+ tables_resolved=include_tables,
+ child_schemas=build(schema.path) if include_tables else [],
+ child_schemas_resolved=include_tables
+ or schema.path not in children,
+ )
+ for schema in children.get(path, [])
+ ]
+
+ return build(parent_path)
+
+ @staticmethod
+ def _schema_tables(
+ schemas: list[Schema], parent_path: tuple[str, ...] = ()
+ ) -> Iterator[tuple[tuple[str, ...], DataTable]]:
+ for schema in schemas:
+ path = (*parent_path, schema.name)
+ for table in schema.tables:
+ yield path, table
+ yield from DuckDBEngine._schema_tables(schema.child_schemas, path)
+
+ def _find_schema(
+ self, database: str, path: tuple[str, ...]
+ ) -> _CatalogSchema | None:
+ normalized_path = tuple(part.lower() for part in path)
+ return next(
+ (
+ schema
+ for schema in self._get_catalog_schemas(database)
+ if tuple(part.lower() for part in schema.path)
+ == normalized_path
+ ),
+ None,
+ )
+
+ def _table_rows(
+ self, schema: _CatalogSchema, table_name: str | None = None
+ ) -> list[tuple[str, DataTableType]]:
+ predicate = "NOT internal AND database_name = ? AND schema_oid = ?"
+ params: list[str | int] = [schema.database, schema.oid]
+ if table_name is not None:
+ predicate += " AND table_name = ?"
+ params.append(table_name)
+ return self._query_catalog(
+ f"SELECT table_name, 'table' FROM duckdb_tables() WHERE {predicate} "
+ f"UNION ALL SELECT view_name, 'view' FROM duckdb_views() WHERE "
+ + predicate.replace("table_name", "view_name")
+ + " ORDER BY 1",
+ params * 2,
+ )
+
+ def _get_schema_tables(
+ self, schema: _CatalogSchema, *, include_table_details: bool
+ ) -> list[DataTable]:
+ tables = {
+ (schema.database, schema.path, name): self._make_table(
+ name, schema.database, table_type=kind
+ )
+ for name, kind in self._table_rows(schema)
+ }
+ if include_table_details:
+ self._load_table_details(
+ tables,
+ schemas=[schema],
+ database=schema.database,
+ schema_oids={schema.oid},
)
- return Schema(name=name, tables=tables, tables_resolved=include_tables)
+ return list(tables.values())
def get_tables_in_schema(
self,
@@ -295,56 +408,58 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
include_table_details: bool,
schema_path: list[str] | None = None,
) -> list[DataTable]:
- """List table and view names without eagerly loading their columns."""
- del schema_path
- rows = self._query_catalog(
- "SELECT table_name, table_type FROM information_schema.tables "
- "WHERE table_catalog = ? AND table_schema = ? ORDER BY table_name",
- [database, schema],
+ catalog_schema = self._find_schema(
+ database, tuple(schema_path or [schema])
+ )
+ if catalog_schema is None:
+ return []
+ return self._get_schema_tables(
+ catalog_schema, include_table_details=include_table_details
)
- tables = {
- (database, schema, name): self._make_table(
- name,
- database,
- table_type="view" if kind == "VIEW" else "table",
- )
- for name, kind in rows
- }
- if include_table_details:
- self._load_table_details(tables, database=database, schema=schema)
- return list(tables.values())
def _load_table_details(
self,
- tables: dict[tuple[str, str, str], DataTable],
+ tables: dict[tuple[str, tuple[str, ...], str], DataTable],
*,
+ schemas: list[_CatalogSchema],
database: str | None = None,
- schema: str | None = None,
+ schema_oids: set[int] | None = None,
) -> None:
if not tables:
return
+ paths = {
+ (schema.database, schema.oid): schema.path for schema in schemas
+ }
query = (
- "SELECT database_name, schema_name, table_name, column_name, data_type "
+ "SELECT database_name, schema_oid, table_name, column_name, data_type "
"FROM duckdb_columns() WHERE NOT internal"
)
- params: list[str] = []
+ params: list[str | int] = []
if database is not None:
query += " AND database_name = ?"
params.append(database)
- if schema is not None:
- query += " AND schema_name = ?"
- params.append(schema)
+ if schema_oids is not None:
+ if not schema_oids:
+ return
+ query += (
+ " AND schema_oid IN ("
+ + ", ".join("?" for _ in schema_oids)
+ + ")"
+ )
+ params.extend(sorted(schema_oids))
query += (
- " ORDER BY database_name, schema_name, table_name, column_index"
+ " ORDER BY database_name, schema_oid, table_name, column_index"
)
try:
rows = self._query_catalog(query, params)
except Exception:
- # A metadata failure must not hide tables already enumerated.
LOGGER.warning("Failed to get DuckDB columns", exc_info=True)
return
- for db_name, schema_name, table_name, column_name, dtype in rows:
- table = tables.get((db_name, schema_name, table_name))
+ for db_name, oid, table_name, column_name, dtype in rows:
+ path = paths.get((db_name, oid))
+ if path is None:
+ continue
+ table = tables.get((db_name, path, table_name))
if table is not None:
table.columns.append(
DataTableColumn(
@@ -354,11 +469,11 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
sample_values=[],
)
)
- for (_, schema_name, _), table in tables.items():
+ for (_, path, _), table in tables.items():
# Some Iceberg catalogs expose a placeholder instead of columns.
if len(table.columns) == 1 and table.columns[0].name == "__":
table.columns = []
- self._describe_table(table, schema_name)
+ self._describe_table(table, path)
if table.columns:
table.num_columns = len(table.columns)
@@ -388,29 +503,30 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
schema_path: list[str] | None = None,
) -> DataTable | None:
"""Describe only the requested table, including remote catalog tables."""
- del schema_path
- rows = self._query_catalog(
- "SELECT table_type FROM information_schema.tables "
- "WHERE table_catalog = ? AND table_schema = ? AND table_name = ?",
- [database_name, schema_name, table_name],
- )
+ path = tuple(schema_path or [schema_name])
+ schema = self._find_schema(database_name, path)
+ if schema is None:
+ return None
+ rows = self._table_rows(schema, table_name)
if not rows:
return None
table = self._make_table(
- table_name,
- database_name,
- table_type="view" if rows[0][0] == "VIEW" else "table",
+ table_name, database_name, table_type=rows[0][1]
)
- self._describe_table(table, schema_name)
+ self._describe_table(table, path)
return table if table.columns else None
- def _describe_table(self, table: DataTable, schema: str) -> None:
+ def _describe_table(
+ self, table: DataTable, schema_path: tuple[str, ...]
+ ) -> None:
import duckdb
connection = cast(
duckdb.DuckDBPyConnection, self._connection or duckdb
)
- qualified_name = quote_qualified_name(table.source, schema, table.name)
+ qualified_name = quote_qualified_name(
+ table.source, *schema_path, table.name
+ )
with self._install_connection(connection):
table.columns = get_table_columns(connection, qualified_name)
if table.columns:
diff --git a/tests/_data/test_sql_summaries.py b/tests/_data/test_sql_summaries.py
index 8397875dd4..f74e789e70 100644
--- a/tests/_data/test_sql_summaries.py
+++ b/tests/_data/test_sql_summaries.py
@@ -160,3 +160,56 @@ def test_geometry_column_type_and_stats():
assert summary.unique is None
finally:
duckdb.execute("DROP TABLE geo_stats_table")
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+def test_nested_column_type() -> None:
+ import duckdb
+
+ from marimo._sql.utils import is_duckdb_v2
+
+ if not is_duckdb_v2():
+ pytest.skip("Nested schemas require DuckDB 2")
+ try:
+ duckdb.execute(
+ "CREATE SCHEMA type_test; CREATE SCHEMA type_test.reports; "
+ "CREATE TABLE type_test.reports.sales (amount INT); "
+ 'CREATE SCHEMA "type_test.reports"; '
+ 'CREATE TABLE "type_test.reports".sales (amount VARCHAR)'
+ )
+ assert (
+ get_column_type('"memory"."type_test"."reports"."sales"', "amount")
+ == "integer"
+ )
+ assert (
+ get_column_type('"memory"."type_test.reports"."sales"', "amount")
+ == "string"
+ )
+ finally:
+ duckdb.execute(
+ 'DROP SCHEMA IF EXISTS type_test CASCADE; DROP SCHEMA IF EXISTS "type_test.reports" CASCADE'
+ )
+
+
[email protected](not HAS_DEPS, reason="optional dependencies not installed")
+ ("schema", "table"),
+ [('column_type_"schema', "t.x"), ("column_type_schema", 't"x')],
+)
+def test_column_type_with_quoted_identifiers(schema: str, table: str) -> None:
+ import duckdb
+
+ from marimo._sql.sql_quoting import (
+ quote_qualified_name,
+ quote_sql_identifier,
+ )
+
+ qualified = quote_qualified_name("memory", schema, table)
+ try:
+ duckdb.execute(f"CREATE SCHEMA {quote_sql_identifier(schema)}")
+ duckdb.execute(f'CREATE TABLE {qualified} ("c""x" INT)')
+ assert get_column_type(qualified, 'c"x') == "integer"
+ finally:
+ duckdb.execute(
+ f"DROP SCHEMA IF EXISTS memory.{quote_sql_identifier(schema)} CASCADE"
+ )
diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.py
index 82e74991a2..aa609bd3f7 100644
--- a/tests/_runtime/test_runtime_datasets.py
+++ b/tests/_runtime/test_runtime_datasets.py
@@ -84,6 +84,43 @@ async def connection_requests() -> list[ExecuteCellCommand]:
@pytest.mark.skipif(not HAS_SQL, reason="SQL deps not available")
class TestPreviewSQLTable:
+ async def test_internal_duckdb_nested_table(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ import duckdb
+
+ from marimo._sql.utils import is_duckdb_v2
+
+ if not is_duckdb_v2():
+ pytest.skip("Nested schemas require DuckDB 2")
+ try:
+ duckdb.execute(
+ "CREATE SCHEMA preview_nested; CREATE SCHEMA preview_nested.reports; CREATE TABLE preview_nested.reports.sales (amount INT)"
+ )
+ await mocked_kernel.k.handle_message(
+ PreviewSQLTableCommand(
+ request_id=RequestId("nested"),
+ engine=INTERNAL_DUCKDB_ENGINE,
+ database="memory",
+ schema="reports",
+ schema_path=["preview_nested", "reports"],
+ table_name="sales",
+ )
+ )
+ results = [
+ op
+ for op in mocked_kernel.stream.operations
+ if isinstance(op, SQLTablePreviewNotification)
+ ]
+ assert len(results) == 1
+ assert results[0].error is None
+ assert results[0].table is not None
+ assert results[0].table.columns[0].name == "amount"
+ finally:
+ duckdb.execute(
+ "DROP SCHEMA IF EXISTS memory.preview_nested CASCADE"
+ )
+
async def test_non_existent_engine(
self, mocked_kernel: MockedKernel
) -> None:
diff --git a/tests/_sql/test_duckdb.py b/tests/_sql/test_duckdb.py
index 17886d6342..70d25249a3 100644
--- a/tests/_sql/test_duckdb.py
+++ b/tests/_sql/test_duckdb.py
@@ -511,8 +511,7 @@ def test_duckdb_discovery_depth(
schemas_enabled
)
assert (
- any("information_schema.tables" in sql for sql in queries)
- == tables_enabled
+ any("duckdb_tables()" in sql for sql in queries) == tables_enabled
)
assert sum("duckdb_columns()" in sql for sql in queries) == int(
details_enabled
@@ -813,7 +812,10 @@ def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:
"Column metadata unavailable"
)
if failure == "placeholder":
- return [("memory", "main", "example", "__", "INTEGER")]
+ oid = connection.execute(
+ "SELECT oid FROM duckdb_schemas() WHERE database_name = 'memory' AND schema_name = 'main'"
+ ).fetchone()[0]
+ return [("memory", oid, "example", "__", "INTEGER")]
return []
return query(sql, params)
@@ -840,3 +842,90 @@ def test_duckdb_keeps_tables_when_column_discovery_fails(failure: str) -> None:
)
]
assert describe.call_count == int(failure == "placeholder")
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
[email protected]("include_tables", [False, True])
+def test_duckdb_nested_schema_discovery(include_tables: bool) -> None:
+ import duckdb
+
+ from marimo._sql.utils import is_duckdb_v2
+
+ if not is_duckdb_v2():
+ pytest.skip("Nested schemas require DuckDB 2")
+ with duckdb.connect() as connection:
+ for parent in ("finance", "other", "finance.reports"):
+ connection.execute(f"CREATE SCHEMA {quote_qualified_name(parent)}")
+ for parent, dtype in (("finance", "INTEGER"), ("other", "VARCHAR")):
+ connection.execute(
+ f"CREATE SCHEMA {quote_qualified_name(parent, 'reports')}"
+ )
+ connection.execute(
+ f"CREATE TABLE {quote_qualified_name(parent, 'reports', 'sales')} (amount {dtype})"
+ )
+ connection.execute(
+ f"CREATE VIEW {quote_qualified_name(parent, 'reports', 'summary')} AS SELECT * FROM {quote_qualified_name(parent, 'reports', 'sales')}"
+ )
+ connection.execute(
+ 'CREATE TABLE "finance.reports".sales (literal BOOLEAN)'
+ )
+ engine = DuckDBEngine(connection)
+ databases = engine.get_databases(
+ include_schemas=True,
+ include_tables=include_tables,
+ include_table_details=True,
+ )
+ schemas = {
+ s.name: s
+ for db in databases
+ if db.name == "memory"
+ for s in db.schemas
+ }
+ assert set(schemas) == {"finance", "other", "finance.reports", "main"}
+ assert schemas["finance"].child_schemas_resolved == include_tables
+ assert schemas["finance"].tables == []
+ for parent, dtype in (("finance", "INTEGER"), ("other", "VARCHAR")):
+ children = (
+ schemas[parent].child_schemas
+ if include_tables
+ else engine.get_schemas(
+ database="memory",
+ schema_path=[parent],
+ include_tables=True,
+ include_table_details=True,
+ )
+ )
+ assert len(children) == 1
+ assert children[0].name == "reports"
+ assert [(t.name, t.type) for t in children[0].tables] == [
+ ("sales", "table"),
+ ("summary", "view"),
+ ]
+ assert all(
+ t.columns[0].external_type == dtype for t in children[0].tables
+ )
+ for name in ("sales", "summary"):
+ details = engine.get_table_details(
+ database_name="memory",
+ schema_name="reports",
+ schema_path=[parent, "reports"],
+ table_name=name,
+ )
+ assert details is not None
+ assert details.columns[0].external_type == dtype
+ literal = engine.get_tables_in_schema(
+ database="memory",
+ schema="finance.reports",
+ schema_path=["finance.reports"],
+ include_table_details=True,
+ )
+ assert literal[0].columns[0].name == "literal"
+ assert (
+ engine.get_tables_in_schema(
+ database="memory",
+ schema="reports",
+ schema_path=["missing", "reports"],
+ include_table_details=True,
+ )
+ == []
+ )