Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(sql): support nested schemas for data sources (#9837) (#9845)

PA06b152af2256
diff --git a/frontend/src/components/datasources/__tests__/filter-empty.test.ts b/frontend/src/components/datasources/__tests__/filter-empty.test.ts
index 5d157ae9ea..3ccec0a467 100644
--- a/frontend/src/components/datasources/__tests__/filter-empty.test.ts
+++ b/frontend/src/components/datasources/__tests__/filter-empty.test.ts
@@ -28,11 +28,15 @@ function makeSchema(opts: {
name: string;
tables: DataTable[];
tables_resolved?: boolean;
+ child_schemas?: DatabaseSchema[];
+ child_schemas_resolved?: boolean;
}): DatabaseSchema {
return {
name: opts.name,
tables: opts.tables,
tables_resolved: opts.tables_resolved ?? true,
+ child_schemas: opts.child_schemas ?? [],
+ child_schemas_resolved: opts.child_schemas_resolved ?? true,
};
}
@@ -180,4 +184,81 @@ describe("filterEmptyDatabases", () => {
expect(databases).toEqual(snapshot);
});
+
+ it("keeps a namespace that has only child namespaces (no own tables)", () => {
+ const databases = [
+ makeDatabase("iceberg", [
+ makeSchema({
+ name: "top",
+ tables: [],
+ child_schemas: [
+ makeSchema({ name: "nested", tables: [makeTable("t1")] }),
+ ],
+ }),
+ ]),
+ ];
+
+ expect(filterEmptyDatabases(databases)).toBe(databases);
+ });
+
+ it("preserves a namespace whose child schemas are deferred", () => {
+ const databases = [
+ makeDatabase("iceberg", [
+ makeSchema({
+ name: "top",
+ tables: [],
+ child_schemas: [],
+ child_schemas_resolved: false,
+ }),
+ ]),
+ ];
+
+ expect(filterEmptyDatabases(databases)).toBe(databases);
+ });
+
+ it("hides a nested namespace that is resolved-empty", () => {
+ const databases = [
+ makeDatabase("iceberg", [
+ makeSchema({
+ name: "top",
+ tables: [makeTable("t1")],
+ child_schemas: [
+ makeSchema({ name: "empty_child", tables: [] }),
+ makeSchema({ name: "full_child", tables: [makeTable("t2")] }),
+ ],
+ }),
+ ]),
+ ];
+
+ expect(filterEmptyDatabases(databases)).toEqual([
+ makeDatabase("iceberg", [
+ makeSchema({
+ name: "top",
+ tables: [makeTable("t1")],
+ child_schemas: [
+ makeSchema({ name: "full_child", tables: [makeTable("t2")] }),
+ ],
+ }),
+ ]),
+ ]);
+ });
+
+ it("hides a parent namespace when all its descendants are empty", () => {
+ const databases = [
+ makeDatabase("iceberg", [
+ makeSchema({
+ name: "top",
+ tables: [],
+ child_schemas: [makeSchema({ name: "empty_child", tables: [] })],
+ }),
+ makeSchema({ name: "other", tables: [makeTable("t1")] }),
+ ]),
+ ];
+
+ expect(filterEmptyDatabases(databases)).toEqual([
+ makeDatabase("iceberg", [
+ makeSchema({ name: "other", tables: [makeTable("t1")] }),
+ ]),
+ ]);
+ });
});
diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.ts
index b3e7c35121..1d7483eb91 100644
--- a/frontend/src/components/datasources/__tests__/utils.test.ts
+++ b/frontend/src/components/datasources/__tests__/utils.test.ts
@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";
import type { SQLTableContext } from "@/core/datasets/data-source-connections";
import { DUCKDB_ENGINE } from "@/core/datasets/engines";
import type { DataTable, DataTableColumn } from "@/core/kernel/messages";
-import { sqlCode } from "../utils";
+import { sqlCode, tableUniqueId } from "../utils";
describe("sqlCode", () => {
const mockTable: DataTable = {
@@ -458,3 +458,64 @@ describe("sqlCode", () => {
});
});
});
+
+describe("tableUniqueId", () => {
+ const ctx = (
+ overrides: Partial<SQLTableContext> & { database: string; schema: string },
+ ): SQLTableContext => ({
+ engine: "e",
+ dialect: "duckdb",
+ ...overrides,
+ });
+
+ it("returns just the table name without a context", () => {
+ expect(tableUniqueId(undefined, "t")).toBe("t");
+ });
+
+ it("uses database + schema for flat engines", () => {
+ expect(tableUniqueId(ctx({ database: "db", schema: "public" }), "t")).toBe(
+ "db.public.t",
+ );
+ });
+
+ it("does not duplicate the leaf schema for nested namespaces", () => {
+ // Regression: previously produced "top.nested.nested.t".
+ expect(
+ tableUniqueId(
+ ctx({ database: "top", schema: "nested", schemaPath: ["nested"] }),
+ "t",
+ ),
+ ).toBe("top.nested.t");
+ });
+
+ it("includes the full schema path for deeply nested namespaces", () => {
+ expect(
+ tableUniqueId(
+ ctx({
+ database: "top",
+ schema: "deep",
+ schemaPath: ["nested", "deep"],
+ }),
+ "t",
+ ),
+ ).toBe("top.nested.deep.t");
+ });
+
+ it("falls back to the flat schema when schemaPath is empty", () => {
+ expect(
+ tableUniqueId(
+ ctx({ database: "db", schema: "public", schemaPath: [] }),
+ "t",
+ ),
+ ).toBe("db.public.t");
+ });
+
+ it("does not emit a double dot for schemaless tables", () => {
+ expect(
+ tableUniqueId(ctx({ database: "db", schema: "", schemaPath: [] }), "t"),
+ ).toBe("db.t");
+ expect(tableUniqueId(ctx({ database: "db", schema: "" }), "t")).toBe(
+ "db.t",
+ );
+ });
+});
diff --git a/frontend/src/components/datasources/components.tsx b/frontend/src/components/datasources/components.tsx
index 08172365c4..abc538a330 100644
--- a/frontend/src/components/datasources/components.tsx
+++ b/frontend/src/components/datasources/components.tsx
@@ -14,25 +14,31 @@ export const RotatingChevron: React.FC<{ isExpanded: boolean }> = ({
export const DatasourceLabel: React.FC<{
children: React.ReactNode;
className?: string;
-}> = ({ children, className }) => {
+ style?: CSSProperties;
+}> = ({ children, className, style }) => {
return (
<div
className={cn(
"flex gap-1.5 items-center font-bold py-1.5 text-muted-foreground bg-(--slate-2) text-sm",
className,
)}
+ style={style}
>
{children}
</div>
);
};
-export const EmptyState: React.FC<{ content: string; className?: string }> = ({
- content,
- className,
-}) => {
+export const EmptyState: React.FC<{
+ content: string;
+ className?: string;
+ style?: CSSProperties;
+}> = ({ content, className, style }) => {
return (
- <div className={cn("text-sm text-muted-foreground py-1", className)}>
+ <div
+ className={cn("text-sm text-muted-foreground py-1", className)}
+ style={style}
+ >
{content}
</div>
);
@@ -61,13 +67,15 @@ export const ErrorState: React.FC<{
export const LoadingState: React.FC<{
message: string;
className?: string;
-}> = ({ message, className }) => {
+ style?: CSSProperties;
+}> = ({ message, className, style }) => {
return (
<div
className={cn(
"text-sm bg-blue-50 dark:bg-(--accent) text-blue-500 dark:text-blue-50 flex items-center gap-2 p-2 h-8",
className,
)}
+ style={style}
>
<LoaderCircle className="h-4 w-4 animate-spin" />
{message}
diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsx
index 3715a2f389..533ce06c51 100644
--- a/frontend/src/components/datasources/datasources.tsx
+++ b/frontend/src/components/datasources/datasources.tsx
@@ -52,6 +52,7 @@ import { useRequestClient } from "@/core/network/requests";
import { variablesAtom } from "@/core/variables/state";
import type { VariableName } from "@/core/variables/types";
import { useAsyncData } from "@/hooks/useAsyncData";
+import { useDeepCompareMemoize } from "@/hooks/useDeepCompareMemoize";
import { sortBy } from "@/utils/arrays";
import { logNever } from "@/utils/assertNever";
import { cn } from "@/utils/cn";
@@ -77,24 +78,44 @@ import {
LoadingState,
RotatingChevron,
} from "./components";
-import { isSchemaless, sqlCode } from "./utils";
+import {
+ areChildSchemasResolved,
+ areSchemasResolved,
+ areTablesResolved,
+ isSchemaless,
+ sqlCode,
+ tableUniqueId,
+} from "./utils";
+
+const INDENT_STEP = 1; // rem per schema nesting level (depth 0 = top-level)
+
+// Indentation (rem) for a schema and its contents at a given nesting depth.
+// Depth 0 is a top-level schema; schemaless tables/columns reuse depth 0 too.
+function schemaHeaderIndentRem(depth: number): number {
+ return 1.75 + depth * INDENT_STEP;
+}
+function schemaTableIndentRem(depth: number): number {
+ return 3 + depth * INDENT_STEP;
+}
+function schemaColumnIndentRem(depth: number): number {
+ return 3.25 + depth * INDENT_STEP;
+}
-// Indentation classes for the datasource tree hierarchy.
+// Left indentation (rem) for each fixed (non-nested) level of the tree.
const INDENT = {
- engineEmpty: "pl-3",
- engine: "pl-3 pr-2",
- database: "pl-4",
- schemaEmpty: "pl-8",
- schema: "pl-7",
- schemaLoading: "pl-8",
- tableLoading: "pl-11",
- tableSchemaless: "pl-8",
- tableWithSchema: "pl-12",
- columnLocal: "pl-5",
- columnSql: "pl-13",
- columnPreview: "pl-10",
+ engineEmpty: 0.75,
+ engine: 0.75,
+ database: 1,
+ tableLoading: 2.75,
+ tableSchemaless: 2,
+ columnLocal: 1.25,
+ columnPreview: 2.5,
};
+function indentStyle(rem: number): React.CSSProperties {
+ return { paddingLeft: `${rem}rem` };
+}
+
const sortedTablesAtom = atom((get) => {
const tables = get(datasetTablesAtom);
const variables = get(variablesAtom);
@@ -132,14 +153,39 @@ export const hideEmptyDatasourcesAtom = atomWithStorage<boolean>(
{ getOnInit: true },
);
-function isKnownEmptySchema(schema: DatabaseSchema): boolean {
- return schema.tables_resolved !== false && schema.tables.length === 0;
+/**
+ * Recursively hide schemas confirmed empty (no tables and no visible child
+ * schemas). Deferred schemas are kept so the user can expand them.
+ */
+function filterEmptySchemas(schemas: DatabaseSchema[]): DatabaseSchema[] {
+ let changed = false;
+ const result: DatabaseSchema[] = [];
+ for (const schema of schemas) {
+ if (!areTablesResolved(schema) || !areChildSchemasResolved(schema)) {
+ result.push(schema);
+ continue;
+ }
+ const childSchemas = schema.child_schemas ?? [];
+ const visibleChildren = filterEmptySchemas(childSchemas);
+ if (schema.tables.length === 0 && visibleChildren.length === 0) {
+ changed = true;
+ continue;
+ }
+ if (visibleChildren === childSchemas) {
+ result.push(schema);
+ continue;
+ }
+ changed = true;
+ result.push({ ...schema, child_schemas: visibleChildren });
+ }
+ return changed ? result : schemas;
}
/**
* Apply the "hide empty" filter to a connection's databases.
*
- * - Schemas with confirmed-empty table lists are hidden.
+ * - Schemas with confirmed-empty table lists (and no child schemas) are
+ * hidden, recursively.
* - Databases are hidden when either (a) their schemas have been enumerated
* and the list is empty, or (b) every schema in them was hidden by the
* schema-level filter.
@@ -152,7 +198,7 @@ export function filterEmptyDatabases(databases: Database[]): Database[] {
const result: Database[] = [];
for (const database of databases) {
// Known-empty database: schema list was enumerated and is empty.
- if (database.schemas_resolved !== false && database.schemas.length === 0) {
+ if (areSchemasResolved(database) && database.schemas.length === 0) {
changed = true;
continue;
}
@@ -161,14 +207,12 @@ export function filterEmptyDatabases(databases: Database[]): Database[] {
result.push(database);
continue;
}
- const visibleSchemas = database.schemas.filter(
- (schema) => !isKnownEmptySchema(schema),
- );
+ const visibleSchemas = filterEmptySchemas(database.schemas);
if (visibleSchemas.length === 0) {
changed = true;
continue;
}
- if (visibleSchemas.length === database.schemas.length) {
+ if (visibleSchemas === database.schemas) {
result.push(database);
continue;
}
@@ -312,29 +356,19 @@ export const DataSources: React.FC = () => {
hasChildren={connection.databases.length > 0}
>
{connection.databases.map((database) => (
- <DatabaseItem
+ <DatabaseTree
key={database.name}
- engineName={connection.name}
+ connection={connection}
database={database}
hasSearch={hasSearch}
- >
- <SchemaList
- schemas={database.schemas}
- defaultSchema={connection.default_schema}
- defaultDatabase={connection.default_database}
- engineName={connection.name}
- databaseName={database.name}
- hasSearch={hasSearch}
- searchValue={searchValue}
- dialect={connection.dialect}
- />
- </DatabaseItem>
+ searchValue={searchValue}
+ />
))}
</Engine>
))}
{dataConnections.length > 0 && tables.length > 0 && (
- <DatasourceLabel className={INDENT.engine}>
+ <DatasourceLabel className="pr-2" style={indentStyle(INDENT.engine)}>
<PythonIcon className="h-4 w-4 text-muted-foreground" />
<span className="text-xs">Python</span>
</DatasourceLabel>
@@ -365,7 +399,7 @@ const Engine: React.FC<{
return (
<>
- <DatasourceLabel className={INDENT.engine}>
+ <DatasourceLabel className="pr-2" style={indentStyle(INDENT.engine)}>
<DatabaseLogo
className="h-4 w-4 text-muted-foreground"
name={connection.dialect}
@@ -388,13 +422,96 @@ const Engine: React.FC<{
) : (
<EmptyState
content="No databases available"
- className={INDENT.engineEmpty}
+ style={indentStyle(INDENT.engineEmpty)}
/>
)}
</>
);
};
+interface DataSourceTree {
+ defaultSchema?: string | null;
+ defaultDatabase?: string | null;
+ dialect: string;
+ engineName: string;
+ databaseName: string;
+ hasSearch: boolean;
+ searchValue?: string;
+}
+
+const DataSourceTreeContext = React.createContext<DataSourceTree | null>(null);
+
+function useDataSourceTree(): DataSourceTree {
+ const tree = React.useContext(DataSourceTreeContext);
+ if (tree == null) {
+ throw new Error(
+ "useDataSourceTree must be used within a DataSourceTreeContext.Provider",
+ );
+ }
+ return tree;
+}
+
+// Build the table context for a (possibly schemaless) schema
+function buildSqlTableContext(
+ tree: DataSourceTree,
+ { schema, schemaPath }: { schema: string; schemaPath: string[] },
+): SQLTableContext {
+ return {
+ engine: tree.engineName,
+ database: tree.databaseName,
+ schema,
+ schemaPath,
+ defaultSchema: tree.defaultSchema,
+ defaultDatabase: tree.defaultDatabase,
+ dialect: tree.dialect,
+ };
+}
+
+const DatabaseTree: React.FC<{
+ connection: DataSourceConnection;
+ database: Database;
+ hasSearch: boolean;
+ searchValue?: string;
+}> = ({ connection, database, hasSearch, searchValue }) => {
+ const tree = React.useMemo<DataSourceTree>(
+ () => ({
+ engineName: connection.name,
+ databaseName: database.name,
+ dialect: connection.dialect,
+ defaultSchema: connection.default_schema,
+ defaultDatabase: connection.default_database,
+ hasSearch,
+ searchValue,
+ }),
+ [
+ connection.name,
+ connection.dialect,
+ connection.default_schema,
+ connection.default_database,
+ database.name,
+ hasSearch,
+ searchValue,
+ ],
+ );
+
+ return (
+ <DatabaseItem
+ engineName={connection.name}
+ database={database}
+ hasSearch={hasSearch}
+ >
+ <DataSourceTreeContext.Provider value={tree}>
+ <SchemaList
+ schemas={database.schemas}
+ schemasResolved={areSchemasResolved(database)}
+ schemaPath={[]}
+ depth={0}
+ />
+ </DataSourceTreeContext.Provider>
+ </DatabaseItem>
+ );
+};
+
const DatabaseItem: React.FC<{
hasSearch: boolean;
engineName: string;
@@ -413,10 +530,8 @@ const DatabaseItem: React.FC<{
return (
<>
<CommandItem
- className={cn(
- "text-sm flex flex-row gap-1 items-center cursor-pointer rounded-none",
- INDENT.database,
- )}
+ className="text-sm flex flex-row gap-1 items-center cursor-pointer rounded-none"
+ style={indentStyle(INDENT.database)}
onSelect={() => {
setIsExpanded(!isExpanded);
setIsSelected(!isSelected);
@@ -441,40 +556,35 @@ const DatabaseItem: React.FC<{
);
};
-const SchemaList: React.FC<{
+interface SchemaListProps {
schemas: DatabaseSchema[];
- defaultSchema?: string | null;
- defaultDatabase?: string | null;
- dialect: string;
- engineName: string;
- databaseName: string;
- hasSearch: boolean;
- searchValue?: string;
-}> = ({
- schemas,
- defaultSchema,
- defaultDatabase,
- dialect,
- engineName,
- databaseName,
- hasSearch,
- searchValue,
-}) => {
+ schemasResolved: boolean;
+ // Parent schema path (relative to the database). Empty at the top level.
+ schemaPath: string[];
+ // Nesting depth (0 = top-level).
+ depth: number;
+}
+
+const SchemaList: React.FC<SchemaListProps> = (props) => {
+ const { schemas, schemasResolved, depth } = props;
+ const tree = useDataSourceTree();
+ const { engineName, databaseName, searchValue } = tree;
const { addSchemaList } = useDataSourceActions();
- const [schemasRequested, setSchemasRequested] = React.useState(false);
+ // Stable identity so the useAsyncData below doesn't refire each render.
+ const schemaPath = useDeepCompareMemoize(props.schemaPath);
// Custom loading state, we need to wait for the data to propagate once requested
// useAsyncData's loading state may return false before data has propagated
const [schemasLoading, setSchemasLoading] = React.useState(false);
const { isPending, error } = useAsyncData(async () => {
- if (schemas.length === 0 && engineName && !schemasRequested) {
- setSchemasRequested(true);
+ if (!schemasResolved && engineName) {
setSchemasLoading(true);
try {
const previewSchemaList = await PreviewSQLSchemaList.request({
engine: engineName,
database: databaseName,
+ schemaPath: schemaPath,
});
addSchemaList({
@@ -482,93 +592,82 @@ const SchemaList: React.FC<{
sqlSchemaContext: {
engine: engineName,
database: databaseName,
+ schemaPath: schemaPath,
},
});
} finally {
setSchemasLoading(false);
}
}
- }, [schemas.length, engineName, databaseName, schemasRequested]);
+ }, [schemasResolved, engineName, databaseName, schemaPath]);
+
+ const stateStyle = indentStyle(schemaHeaderIndentRem(depth));
if (isPending || schemasLoading) {
- return (
- <LoadingState
- message="Loading schemas..."
- className={INDENT.schemaLoading}
- />
- );
+ return <LoadingState message="Loading schemas..." style={stateStyle} />;
}
if (error) {
- return <ErrorState error={error} className={INDENT.schemaLoading} />;
+ return <ErrorState error={error} style={stateStyle} />;
}
if (schemas.length === 0) {
- return (
- <EmptyState
- content="No schemas available"
- className={INDENT.schemaEmpty}
- />
- );
+ return <EmptyState content="No schemas available" style={stateStyle} />;
}
- const filteredSchemas = schemas.filter((schema) => {
- if (searchValue) {
- return schema.tables.some((table) =>
- table.name.toLowerCase().includes(searchValue.toLowerCase()),
- );
- }
- return true;
- });
-
return (
<>
- {filteredSchemas.map((schema) => (
- <SchemaItem
- key={schema.name}
- databaseName={databaseName}
- schema={schema}
- hasSearch={hasSearch}
- >
- <TableList
- tables={schema.tables}
- searchValue={searchValue}
- sqlTableContext={{
- engine: engineName,
- database: databaseName,
- schema: schema.name,
- defaultSchema: defaultSchema,
- defaultDatabase: defaultDatabase,
- dialect: dialect,
- }}
+ {schemas.map((schema) => {
+ // Schemaless schemas (the database's own tables) render their tables
+ // directly under the database with no expandable node.
+ if (isSchemaless(schema.name)) {
+ return (
+ <TableList
+ key={schema.name}
+ tables={schema.tables}
+ tablesResolved={areTablesResolved(schema)}
+ searchValue={searchValue}
+ sqlTableContext={buildSqlTableContext(tree, {
+ schema: schema.name,
+ schemaPath,
+ })}
+ />
+ );
+ }
+ return (
+ <SchemaNode
+ key={schema.name}
+ schema={schema}
+ schemaPath={[...schemaPath, schema.name]}
+ depth={depth}
/>
- </SchemaItem>
- ))}
+ );
+ })}
</>
);
};
-const SchemaItem: React.FC<{
- databaseName: string;
+interface SchemaNodeProps {
schema: DatabaseSchema;
- children: React.ReactNode;
- hasSearch: boolean;
-}> = ({ databaseName, schema, children, hasSearch }) => {
+ // Path of this schema relative to the database (includes this node).
+ schemaPath: string[];
+ depth: number;
+}
+
+const SchemaNode: React.FC<SchemaNodeProps> = (props) => {
+ const { schema, schemaPath, depth } = props;
+ const tree = useDataSourceTree();
+ const { databaseName, hasSearch, searchValue } = tree;
const [isExpanded, setIsExpanded] = React.useState(hasSearch);
const [isSelected, setIsSelected] = React.useState(false);
- const uniqueValue = `${databaseName}:${schema.name}`;
-
- if (isSchemaless(schema.name)) {
- return children;
- }
+ const uniqueValue = `${databaseName}:${schemaPath.join(".")}`;
+ const childSchemas = schema.child_schemas ?? [];
return (
<>
<CommandItem
- className={cn(
- "text-sm flex flex-row gap-1 items-center cursor-pointer rounded-none",
- INDENT.schema,
- )}
+ className="text-sm flex flex-row gap-1 items-center cursor-pointer rounded-none"
+ style={indentStyle(schemaHeaderIndentRem(depth))}
onSelect={() => {
setIsExpanded(!isExpanded);
setIsSelected(!isSelected);
@@ -586,7 +685,31 @@ const SchemaItem: React.FC<{
{schema.name}
</span>
</CommandItem>
- {isExpanded && children}
+ {isExpanded && (
+ <>
+ {/* Nested child schemas */}
+ {(childSchemas.length > 0 || !areChildSchemasResolved(schema)) && (
+ <SchemaList
+ schemas={childSchemas}
+ schemasResolved={areChildSchemasResolved(schema)}
+ schemaPath={schemaPath}
+ depth={depth + 1}
+ />
+ )}
+ {/* Tables that live directly in this schema */}
+ <TableList
+ tables={schema.tables}
+ tablesResolved={areTablesResolved(schema)}
+ searchValue={searchValue}
+ tableIndentRem={schemaTableIndentRem(depth)}
+ columnIndentRem={schemaColumnIndentRem(depth)}
+ sqlTableContext={buildSqlTableContext(tree, {
+ schema: schema.name,
+ schemaPath,
+ })}
+ />
+ </>
+ )}
</>
);
};
@@ -595,24 +718,39 @@ const TableList: React.FC<{
tables: DataTable[];
sqlTableContext?: SQLTableContext;
searchValue?: string;
-}> = ({ tables, sqlTableContext, searchValue }) => {
+ // Whether `tables` has been enumerated; when false, discovery is deferred and
+ // a request is issued on mount (i.e. when the parent is expanded).
+ tablesResolved?: boolean;
+ // Depth-based indentation (rem) for nested schema tables/columns. When
+ // omitted, the fixed INDENT levels are used (top-level / schemaless tables).
+ tableIndentRem?: number;
+ columnIndentRem?: number;
+}> = ({
+ tables,
+ sqlTableContext,
+ searchValue,
+ tablesResolved = true,
+ tableIndentRem,
+ columnIndentRem,
+}) => {
const { addTableList } = useDataSourceActions();
- const [tablesRequested, setTablesRequested] = React.useState(false);
// Custom loading state, we need to wait for the data to propagate once requested
// useAsyncData's loading state may return false before data has propagated
const [tablesLoading, setTablesLoading] = React.useState(false);
+ // Fetch when discovery is deferred (also re-fetches after a refresh, which
+ // resets tablesResolved to false).
const { isPending, error } = useAsyncData(async () => {
- if (tables.length === 0 && sqlTableContext && !tablesRequested) {
- setTablesRequested(true);
+ if (!tablesResolved && sqlTableContext) {
setTablesLoading(true);
- const { engine, database, schema } = sqlTableContext;
+ const { engine, database, schema, schemaPath } = sqlTableContext;
const previewTableList = await PreviewSQLTableList.request({
engine: engine,
database: database,
schema: schema,
+ schemaPath: schemaPath ?? [],
});
if (!previewTableList?.tables) {
@@ -626,25 +764,20 @@ const TableList: React.FC<{
});
setTablesLoading(false);
}
- }, [tables.length, sqlTableContext, tablesRequested]);
+ }, [tablesResolved, sqlTableContext]);
+
+ const stateStyle = indentStyle(tableIndentRem ?? INDENT.tableLoading);
if (isPending || tablesLoading) {
- return (
- <LoadingState
- message="Loading tables..."
- className={INDENT.tableLoading}
- />
- );
+ return <LoadingState message="Loading tables..." style={stateStyle} />;
}
if (error) {
- return <ErrorState error={error} className={INDENT.tableLoading} />;
+ return <ErrorState error={error} style={stateStyle} />;
}
if (tables.length === 0) {
- return (
- <EmptyState content="No tables found" className={INDENT.tableLoading} />
- );
+ return <EmptyState content="No tables found" style={stateStyle} />;
}
const filteredTables = tables.filter((table) => {
@@ -662,6 +795,8 @@ const TableList: React.FC<{
table={table}
sqlTableContext={sqlTableContext}
isSearching={!!searchValue}
+ tableIndentRem={tableIndentRem}
+ columnIndentRem={columnIndentRem}
/>
))}
</>
@@ -672,28 +807,29 @@ const DatasetTableItem: React.FC<{
table: DataTable;
sqlTableContext?: SQLTableContext;
isSearching: boolean;
-}> = ({ table, sqlTableContext, isSearching }) => {
+ tableIndentRem?: number;
+ columnIndentRem?: number;
+}> = ({
+ table,
+ sqlTableContext,
+ isSearching,
+ tableIndentRem,
+ columnIndentRem,
+}) => {
const { addTable } = useDataSourceActions();
const [isExpanded, setIsExpanded] = React.useState(false);
- const [tableDetailsRequested, setTableDetailsRequested] =
- React.useState(false);
const tableDetailsExist = table.columns.length > 0;
const { isFetching, isPending, error } = useAsyncData(async () => {
- if (
- isExpanded &&
- !tableDetailsExist &&
- sqlTableContext &&
- !tableDetailsRequested
- ) {
- setTableDetailsRequested(true);
- const { engine, database, schema } = sqlTableContext;
+ if (isExpanded && !tableDetailsExist && sqlTableContext) {
+ const { engine, database, schema, schemaPath } = sqlTableContext;
const previewTable = await PreviewSQLTable.request({
engine: engine,
database: database,
schema: schema,
tableName: table.name,
+ schemaPath: schemaPath ?? [],
});
if (!previewTable?.table) {
@@ -720,8 +856,14 @@ const DatasetTableItem: React.FC<{
});
const getCode = () => {
if (table.source_type === "catalog") {
+ // Build the fully-qualified, dotted name including any nested
+ // schema path, e.g. `top.nested.table`.
const identifier = sqlTableContext?.database
- ? `${sqlTableContext.database}.${table.name}`
+ ? [
+ sqlTableContext.database,
+ ...(sqlTableContext.schemaPath ?? []),
+ table.name,
+ ].join(".")
: table.name;
return `${table.engine}.load_table("${identifier}")`;
}
@@ -768,28 +910,20 @@ const DatasetTableItem: React.FC<{
};
const renderColumns = () => {
+ const stateStyle = indentStyle(columnIndentRem ?? INDENT.tableLoading);
+
if (isPending || isFetching) {
- return (
- <LoadingState
- message="Loading columns..."
- className={INDENT.tableLoading}
- />
- );
+ return <LoadingState message="Loading columns..." style={stateStyle} />;
}
if (error) {
- return <ErrorState error={error} className={INDENT.tableLoading} />;
+ return <ErrorState error={error} style={stateStyle} />;
}
const columns = table.columns;
if (columns.length === 0) {
- return (
- <EmptyState
- content="No columns found"
- className={INDENT.tableLoading}
- />
- );
+ return <EmptyState content="No columns found" style={stateStyle} />;
}
return columns.map((column) => (
@@ -798,6 +932,7 @@ const DatasetTableItem: React.FC<{
table={table}
column={column}
sqlTableContext={sqlTableContext}
+ columnIndentRem={columnIndentRem}
/>
));
};
@@ -816,21 +951,19 @@ const DatasetTableItem: React.FC<{
);
};
- const uniqueId = sqlTableContext
- ? `${sqlTableContext.database}.${sqlTableContext.schema}.${table.name}`
- : table.name;
+ const uniqueId = tableUniqueId(sqlTableContext, table.name);
+
+ const tableRem =
+ tableIndentRem ?? (sqlTableContext ? INDENT.tableSchemaless : 0);
return (
<>
<CommandItem
className={cn(
"rounded-none group h-8 cursor-pointer",
- sqlTableContext &&
- (isSchemaless(sqlTableContext.schema)
- ? INDENT.tableSchemaless
- : INDENT.tableWithSchema),
(isExpanded || isSearching) && "font-semibold",
)}
+ style={indentStyle(tableRem)}
value={uniqueId}
aria-selected={isExpanded}
forceMount={true}
@@ -861,7 +994,8 @@ const DatasetColumnItem: React.FC<{
table: DataTable;
column: DataTableColumn;
sqlTableContext?: SQLTableContext;
-}> = ({ table, column, sqlTableContext }) => {
+ columnIndentRem?: number;
+}> = ({ table, column, sqlTableContext, columnIndentRem }) => {
const [isExpanded, setIsExpanded] = React.useState(false);
const closeAllColumns = useAtomValue(closeAllColumnsAtom);
const setExpandedColumns = useSetAtom(expandedColumnsAtom);
@@ -920,9 +1054,10 @@ const DatasetColumnItem: React.FC<{
onSelect={() => setIsExpanded(!isExpanded)}
>
<div
- className={cn(
- "flex flex-row gap-2 items-center flex-1",
- sqlTableContext ? INDENT.columnSql : INDENT.columnLocal,
+ className="flex flex-row gap-2 items-center flex-1"
+ style={indentStyle(
+ columnIndentRem ??
+ (sqlTableContext ? schemaColumnIndentRem(0) : INDENT.columnLocal),
)}
>
<ColumnName columnName={columnText} dataType={column.type} />
@@ -950,10 +1085,8 @@ const DatasetColumnItem: React.FC<{
</CommandItem>
{isExpanded && (
<div
- className={cn(
- INDENT.columnPreview,
- "pr-2 py-2 bg-(--slate-1) shadow-inner border-b",
- )}
+ className="pr-2 py-2 bg-(--slate-1) shadow-inner border-b"
+ style={indentStyle(INDENT.columnPreview)}
>
<ErrorBoundary>
<DatasetColumnPreview
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.ts
index 113d892805..f75753f506 100644
--- a/frontend/src/components/datasources/utils.ts
+++ b/frontend/src/components/datasources/utils.ts
@@ -4,15 +4,54 @@ import { BigQueryDialect } from "@marimo-team/codemirror-sql/dialects";
import { isKnownDialect } from "@/core/codemirror/language/languages/sql/utils";
import type { SQLTableContext } from "@/core/datasets/data-source-connections";
import { DUCKDB_ENGINE } from "@/core/datasets/engines";
-import type { DataTable, DataType } from "@/core/kernel/messages";
+import type {
+ Database,
+ DatabaseSchema,
+ DataTable,
+ DataType,
+} from "@/core/kernel/messages";
import { logNever } from "@/utils/assertNever";
import type { ColumnHeaderStatsKey } from "../data-table/types";
+/**
+ * Stable id for a table node in the datasources tree.
+ *
+ * schemaPath already includes the leaf schema for nested namespaces, so use it
+ * when present and fall back to the flat schema name otherwise (avoids
+ * duplicating the leaf, e.g. `top.nested.nested.table`).
+ */
+export function tableUniqueId(
+ sqlTableContext: SQLTableContext | undefined,
+ tableName: string,
+): string {
+ if (!sqlTableContext) {
+ return tableName;
+ }
+ const segments = (
+ sqlTableContext.schemaPath?.length
+ ? sqlTableContext.schemaPath
+ : [sqlTableContext.schema]
+ ).filter(Boolean);
+ return [sqlTableContext.database, ...segments, tableName].join(".");
+}
+
// Some databases have no schemas, so we don't show it (eg. Clickhouse)
export function isSchemaless(schemaName: string) {
return schemaName === "";
}
+// Lazy discovery: the `*_resolved` flags default to `true` and are only `false`
+// when enumeration was deferred. Helper functions to centralize logic
+export function areSchemasResolved(database: Database): boolean {
+ return database.schemas_resolved !== false;
+}
+export function areTablesResolved(schema: DatabaseSchema): boolean {
+ return schema.tables_resolved !== false;
+}
+export function areChildSchemasResolved(schema: DatabaseSchema): boolean {
+ return schema.child_schemas_resolved !== false;
+}
+
interface SqlCodeFormatter {
/**
* Format the table path based on dialect-specific rules
diff --git a/frontend/src/core/datasets/__tests__/data-source.test.ts b/frontend/src/core/datasets/__tests__/data-source.test.ts
index 2d72df354d..afc12f6fc8 100644
--- a/frontend/src/core/datasets/__tests__/data-source.test.ts
+++ b/frontend/src/core/datasets/__tests__/data-source.test.ts
@@ -2,9 +2,12 @@
import { beforeEach, describe, expect, it } from "vitest";
import { variableName } from "@/__tests__/branded";
import type { DatabaseSchema, DataTable } from "@/core/kernel/messages";
+import { store } from "@/core/state/jotai";
import type { VariableName } from "@/core/variables/types";
import {
+ allTablesAtom,
type DataSourceConnection,
+ dataSourceConnectionsAtom,
type DataSourceState,
exportedForTesting,
type SQLTableContext,
@@ -575,3 +578,226 @@ describe("add table", () => {
expect(db1?.schemas.length).toBe(1);
});
});
+
+describe("nested namespaces", () => {
+ // Iceberg-style: database "top" with a schemaless schema and a nested
+ // namespace "nested" that has a deferred child namespace "deep".
+ const nestedConnections: DataSourceConnection[] = [
+ {
+ name: "ice" as ConnectionName,
+ source: "iceberg",
+ display_name: "Iceberg",
+ dialect: "iceberg",
+ databases: [
+ {
+ name: "top",
+ dialect: "iceberg",
+ schemas_resolved: true,
+ schemas: [
+ { name: "", tables: [], tables_resolved: true },
+ {
+ name: "nested",
+ tables: [],
+ tables_resolved: false,
+ child_schemas: [],
+ child_schemas_resolved: false,
+ },
+ ],
+ },
+ ],
+ },
+ ];
+
+ let baseState: DataSourceState;
+
+ beforeEach(() => {
+ baseState = addConnection(nestedConnections, initialState());
+ });
+
+ const findSchema = (
+ state: DataSourceState,
+ path: string[],
+ ): DatabaseSchema | undefined => {
+ const conn = state.connectionsMap.get("ice" as ConnectionName);
+ const db = conn?.databases.find((d) => d.name === "top");
+ let schemas = db?.schemas ?? [];
+ let found: DatabaseSchema | undefined;
+ for (const segment of path) {
+ found = schemas.find((s) => s.name === segment);
+ schemas = found?.child_schemas ?? [];
+ }
+ return found;
+ };
+
+ it("sets child namespaces at a nested path", () => {
+ const children: DatabaseSchema[] = [
+ { name: "deep", tables: [], tables_resolved: false },
+ ];
+ const newState = reducer(baseState, {
+ type: "addSchemaList",
+ payload: {
+ schemas: children,
+ sqlSchemaContext: {
+ engine: "ice",
+ database: "top",
+ schemaPath: ["nested"],
+ },
+ },
+ });
+
+ const nested = findSchema(newState, ["nested"]);
+ expect(nested?.child_schemas_resolved).toBe(true);
+ expect(nested?.child_schemas?.map((s) => s.name)).toEqual(["deep"]);
+ // The schemaless sibling is untouched.
+ expect(findSchema(newState, [""])?.name).toBe("");
+ });
+
+ it("sets tables at a nested path", () => {
+ const tables: DataTable[] = [
+ {
+ name: "table4",
+ columns: [],
+ num_columns: 0,
+ num_rows: 0,
+ variable_name: null,
+ source: "iceberg",
+ source_type: "catalog",
+ type: "table",
+ },
+ ];
+ const newState = reducer(baseState, {
+ type: "addTableList",
+ payload: {
+ tables,
+ sqlTableContext: {
+ engine: "ice",
+ database: "top",
+ schema: "nested",
+ dialect: "iceberg",
+ schemaPath: ["nested"],
+ },
+ },
+ });
+
+ const nested = findSchema(newState, ["nested"]);
+ expect(nested?.tables_resolved).toBe(true);
+ expect(nested?.tables.map((t) => t.name)).toEqual(["table4"]);
+ });
+
+ it("replaces a single table at a nested path", () => {
+ const makeTable = (numRows: number): DataTable => ({
+ name: "table4",
+ columns: [],
+ num_columns: 0,
+ num_rows: numRows,
+ variable_name: null,
+ source: "iceberg",
+ source_type: "catalog",
+ type: "table",
+ });
+ const context = {
+ engine: "ice",
+ database: "top",
+ schema: "nested",
+ dialect: "iceberg",
+ schemaPath: ["nested"],
+ };
+ let state = reducer(baseState, {
+ type: "addTableList",
+ payload: { tables: [makeTable(1)], sqlTableContext: context },
+ });
+ state = reducer(state, {
+ type: "addTable",
+ payload: { table: makeTable(42), sqlTableContext: context },
+ });
+
+ const nested = findSchema(state, ["nested"]);
+ expect(nested?.tables).toHaveLength(1);
+ expect(nested?.tables[0].num_rows).toBe(42);
+ });
+
+ it("does not change anything for a missing nested path", () => {
+ const newState = reducer(baseState, {
+ type: "addSchemaList",
+ payload: {
+ schemas: [{ name: "deep", tables: [] }],
+ sqlSchemaContext: {
+ engine: "ice",
+ database: "top",
+ schemaPath: ["does_not_exist"],
+ },
+ },
+ });
+ // The result is unchanged: nested namespace stays unresolved and the
+ // database keeps its two schemas (schemaless + nested).
+ const newDb = newState.connectionsMap
+ .get("ice" as ConnectionName)
+ ?.databases.find((d) => d.name === "top");
+ expect(findSchema(newState, ["nested"])?.child_schemas_resolved).toBe(
+ false,
+ );
+ expect(newDb?.schemas.length).toBe(2);
+ });
+});
+
+describe("allTablesAtom with nested namespaces", () => {
+ it("enumerates tables from nested namespaces", () => {
+ const table = (name: string): DataTable => ({
+ name,
+ columns: [],
+ num_columns: 0,
+ num_rows: 0,
+ variable_name: null,
+ source: "iceberg",
+ source_type: "catalog",
+ type: "table",
+ });
+
+ const state = addConnection(
+ [
+ {
+ name: "ice" as ConnectionName,
+ source: "iceberg",
+ display_name: "Iceberg",
+ dialect: "iceberg",
+ databases: [
+ {
+ name: "top",
+ dialect: "iceberg",
+ schemas_resolved: true,
+ schemas: [
+ {
+ name: "",
+ tables: [table("toptable")],
+ tables_resolved: true,
+ },
+ {
+ name: "nested",
+ tables: [table("nestedtable")],
+ tables_resolved: true,
+ child_schemas_resolved: true,
+ child_schemas: [
+ {
+ name: "deep",
+ tables: [table("deeptable")],
+ tables_resolved: true,
+ child_schemas: [],
+ child_schemas_resolved: true,
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ initialState(),
+ );
+
+ store.set(dataSourceConnectionsAtom, state);
+ const names = [...store.get(allTablesAtom).values()].map((t) => t.name);
+ expect(names).toContain("toptable");
+ expect(names).toContain("nestedtable");
+ expect(names).toContain("deeptable");
+ });
+});
diff --git a/frontend/src/core/datasets/data-source-connections.ts b/frontend/src/core/datasets/data-source-connections.ts
index d205a3f4c3..0f8e0bc8ae 100644
--- a/frontend/src/core/datasets/data-source-connections.ts
+++ b/frontend/src/core/datasets/data-source-connections.ts
@@ -49,6 +49,9 @@ export interface DataSourceState {
export interface SQLSchemaContext {
engine: string;
database: string;
+ // Parent schema path (relative to `database`) for nested schemas.
+ // Empty/undefined for the database's top level.
+ schemaPath?: string[];
}
export interface SQLTableContext {
@@ -58,6 +61,49 @@ export interface SQLTableContext {
dialect: string;
defaultSchema?: string | null;
defaultDatabase?: string | null;
+ // Nested schema path (relative to `database`). Empty/undefined at top level.
+ schemaPath?: string[];
+}
+
+/**
+ * Immutably descend `path` (schema segment names) into a nested schema list
+ * and apply `update` to the matching schema. Unmatched branches are unchanged.
+ */
+function updateSchemaAtPath(
+ schemas: DatabaseSchema[],
+ path: string[],
+ update: (schema: DatabaseSchema) => DatabaseSchema,
+): DatabaseSchema[] {
+ if (path.length === 0) {
+ return schemas;
+ }
+ const [head, ...rest] = path;
+ return schemas.map((schema) => {
+ if (schema.name !== head) {
+ return schema;
+ }
+ if (rest.length === 0) {
+ return update(schema);
+ }
+ return {
+ ...schema,
+ child_schemas: updateSchemaAtPath(
+ schema.child_schemas ?? [],
+ rest,
+ update,
+ ),
+ };
+ });
+}
+
+/**
+ * The path (schema/namespace segment names within a database) that locates the
+ * schema holding a set of tables. For nested namespaces this is the
+ * `schemaPath`; otherwise it is the single (possibly schemaless) schema name.
+ */
+function tableSchemaPath(sqlTableContext: SQLTableContext): string[] {
+ const { schemaPath, schema } = sqlTableContext;
+ return schemaPath && schemaPath.length > 0 ? schemaPath : [schema];
}
function initialState(): DataSourceState {
@@ -159,6 +205,7 @@ const {
return state;
}
+ const schemaPath = sqlSchemaContext.schemaPath ?? [];
const newMap = new Map(connectionsMap);
const newConn: DataSourceConnection = {
...conn,
@@ -166,10 +213,22 @@ const {
if (db.name !== sqlSchemaContext.database) {
return db;
}
+ // Top level: replace the database's schemas.
+ if (schemaPath.length === 0) {
+ return {
+ ...db,
+ schemas: schemas,
+ schemas_resolved: true,
+ };
+ }
+ // Nested namespace: replace the child schemas of the namespace at path.
return {
...db,
- schemas: schemas,
- schemas_resolved: true,
+ schemas: updateSchemaAtPath(db.schemas, schemaPath, (schema) => ({
+ ...schema,
+ child_schemas: schemas,
+ child_schemas_resolved: true,
+ })),
};
}),
};
@@ -198,6 +257,7 @@ const {
return state;
}
+ const path = tableSchemaPath(sqlTableContext);
const newMap = new Map(connectionsMap);
const newConn: DataSourceConnection = {
...conn,
@@ -207,16 +267,11 @@ const {
}
return {
...db,
- schemas: db.schemas.map((schema) => {
- if (schema.name !== sqlTableContext.schema) {
- return schema;
- }
- return {
- ...schema,
- tables: tables,
- tables_resolved: true,
- };
- }),
+ schemas: updateSchemaAtPath(db.schemas, path, (schema) => ({
+ ...schema,
+ tables: tables,
+ tables_resolved: true,
+ })),
};
}),
};
@@ -246,6 +301,7 @@ const {
return state;
}
+ const path = tableSchemaPath(sqlTableContext);
const newMap = new Map(connectionsMap);
const newConn: DataSourceConnection = {
...conn,
@@ -253,21 +309,15 @@ const {
if (db.name !== sqlTableContext.database) {
return db;
}
-
return {
...db,
- schemas: db.schemas.map((schema) => {
- if (schema.name !== sqlTableContext.schema) {
- return schema;
- }
-
+ schemas: updateSchemaAtPath(db.schemas, path, (schema) => {
// If tables array is empty, add the table
// Otherwise, replace existing table or keep unchanged tables
const tables =
schema.tables.length === 0
? [table]
: schema.tables.map((t) => (t.name === tableName ? table : t));
-
return {
...schema,
tables,
@@ -327,9 +377,14 @@ export const allTablesAtom = atom((get) => {
const isDefaultDb =
database.name === conn.default_database || conn.databases.length === 1;
- for (const schema of database.schemas) {
- const isDefaultSchema = schema.name === conn.default_schema;
- const schemalessDb = isSchemaless(schema.name);
+ // Walk schemas recursively so nested namespaces (e.g. Iceberg
+ // `top.nested`) are enumerated. `segments` is the path of named
+ // (non-schemaless) namespace segments down to this schema.
+ const walkSchema = (schema: DatabaseSchema, segments: string[]): void => {
+ const schemalessDb = segments.length === 0;
+ const isDefaultSchema =
+ segments.length === 1 && segments[0] === conn.default_schema;
+ const schemaQualifier = segments.join(".");
for (const table of schema.tables) {
let nameToSave: string;
@@ -358,14 +413,14 @@ export const allTablesAtom = atom((get) => {
continue;
}
- nameToSave = `${schema.name}.${table.name}`;
+ nameToSave = `${schemaQualifier}.${table.name}`;
if (isDefaultDb && !tableNames.has(nameToSave)) {
tableNames.set(nameToSave, table);
continue;
}
- nameToSave = `${database.name}.${schema.name}.${table.name}`;
+ nameToSave = `${database.name}.${schemaQualifier}.${table.name}`;
if (tableNames.has(nameToSave)) {
Logger.warn(`Table name collision for ${nameToSave}. Skipping.`);
@@ -373,6 +428,15 @@ export const allTablesAtom = atom((get) => {
tableNames.set(nameToSave, table);
}
}
+
+ // Recurse into nested child namespaces. Children are always named.
+ for (const child of schema.child_schemas ?? []) {
+ walkSchema(child, [...segments, child.name]);
+ }
+ };
+
+ for (const schema of database.schemas) {
+ walkSchema(schema, isSchemaless(schema.name) ? [] : [schema.name]);
}
}
}
diff --git a/marimo/_data/models.py b/marimo/_data/models.py
index 40fc9a9721..2b5aab5e21 100644
--- a/marimo/_data/models.py
+++ b/marimo/_data/models.py
@@ -4,6 +4,8 @@ from __future__ import annotations
from datetime import date, datetime, time, timedelta # noqa: TC003
from typing import TYPE_CHECKING, Any, Literal
+import msgspec
+
from marimo._types.ids import VariableName
from marimo._utils.msgspec_basestruct import BaseStruct
@@ -88,16 +90,24 @@ class Schema(BaseStruct):
"""
Represents a database schema and its tables.
+ A schema may itself contain nested child schemas, e.g. for catalogs with
+ hierarchical namespaces such as Iceberg (`top.nested.deep`).
+
Attributes:
name (str): The name of the schema.
tables (List[DataTable]): Tables in this schema.
tables_resolved (bool): True when `tables` has been enumerated
False when table discovery was deferred. Defaults to True
+ child_schemas (List[Schema]): Nested child schemas (sub-namespaces).
+ child_schemas_resolved (bool): True when `child_schemas` has been
+ enumerated. False when discovery was deferred. Defaults to True
"""
name: str
tables: list[DataTable]
tables_resolved: bool = True
+ child_schemas: list[Schema] = msgspec.field(default_factory=list)
+ child_schemas_resolved: bool = True
class Database(BaseStruct):
diff --git a/marimo/_messaging/notification.py b/marimo/_messaging/notification.py
index 554f880752..8d52de9149 100644
--- a/marimo/_messaging/notification.py
+++ b/marimo/_messaging/notification.py
@@ -552,10 +552,13 @@ class SQLDatabaseMetadata(msgspec.Struct):
Attributes:
connection: Connection identifier.
database: Database name.
+ schema_path: Parent schema path the schemas belong under. Empty for
+ the database's top level.
"""
connection: str
database: str
+ schema_path: list[str] = msgspec.field(default_factory=list)
class SQLMetadata(msgspec.Struct, tag="sql-metadata"):
@@ -565,11 +568,14 @@ class SQLMetadata(msgspec.Struct, tag="sql-metadata"):
connection: Connection identifier.
database: Database name.
schema: Schema name.
+ schema_path: Path of nested schemas (relative to `database`). Empty
+ for the top level.
"""
connection: str
database: str
schema: str
+ schema_path: list[str] = msgspec.field(default_factory=list)
class SQLTablePreviewNotification(Notification, tag="sql-table-preview"):
diff --git a/marimo/_runtime/callbacks/datasets.py b/marimo/_runtime/callbacks/datasets.py
index a401fdaab3..fb255c4fed 100644
--- a/marimo/_runtime/callbacks/datasets.py
+++ b/marimo/_runtime/callbacks/datasets.py
@@ -161,10 +161,12 @@ class DatasetCallbacks:
database_name = request.database
schema_name = request.schema
table_name = request.table_name
+ schema_path = request.schema_path
sql_metadata = SQLMetadata(
connection=variable_name,
database=database_name,
schema=schema_name,
+ schema_path=schema_path,
)
engine, error = self.get_engine_catalog(variable_name)
@@ -184,6 +186,7 @@ class DatasetCallbacks:
table_name=table_name,
schema_name=schema_name,
database_name=database_name,
+ schema_path=schema_path,
)
broadcast_notification(
@@ -223,10 +226,12 @@ class DatasetCallbacks:
variable_name = cast(VariableName, request.engine)
database_name = request.database
schema_name = request.schema
+ schema_path = request.schema_path
sql_metadata = SQLMetadata(
connection=variable_name,
database=database_name,
schema=schema_name,
+ schema_path=schema_path,
)
engine, error = self.get_engine_catalog(variable_name)
@@ -246,6 +251,7 @@ class DatasetCallbacks:
schema=schema_name,
database=database_name,
include_table_details=False,
+ schema_path=schema_path,
)
broadcast_notification(
SQLTableListPreviewNotification(
@@ -280,9 +286,11 @@ class DatasetCallbacks:
"""
variable_name = cast(VariableName, request.engine)
database_name = request.database
+ schema_path = request.schema_path
sql_db_metadata = SQLDatabaseMetadata(
connection=variable_name,
database=database_name,
+ schema_path=schema_path,
)
engine, error = self.get_engine_catalog(variable_name)
@@ -302,6 +310,7 @@ class DatasetCallbacks:
database=database_name,
include_tables=False,
include_table_details=False,
+ schema_path=schema_path,
)
broadcast_notification(
SQLSchemaListPreviewNotification(
diff --git a/marimo/_runtime/commands.py b/marimo/_runtime/commands.py
index 0b860176c9..a162954202 100644
--- a/marimo/_runtime/commands.py
+++ b/marimo/_runtime/commands.py
@@ -666,6 +666,8 @@ class PreviewSQLTableCommand(Command):
database: Database containing the table.
schema: Schema containing the table.
table_name: Table to preview.
+ schema_path: Path of nested schemas (relative to `database`) for
+ catalogs with nested schemas. Empty for the top level.
"""
request_id: RequestId
@@ -673,6 +675,7 @@ class PreviewSQLTableCommand(Command):
database: str
schema: str
table_name: str
+ schema_path: list[str] = msgspec.field(default_factory=list)
class ListSQLTablesCommand(Command):
@@ -686,12 +689,15 @@ class ListSQLTablesCommand(Command):
engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).
database: Database to query.
schema: Schema to list tables from.
+ schema_path: Path of nested schemas (relative to `database`) for
+ catalogs with nested schemas. Empty for the top level.
"""
request_id: RequestId
engine: str
database: str
schema: str
+ schema_path: list[str] = msgspec.field(default_factory=list)
class ListSQLSchemasCommand(Command):
@@ -704,11 +710,14 @@ class ListSQLSchemasCommand(Command):
request_id: Unique identifier for this request.
engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).
database: Database to query.
+ schema_path: Parent schema path whose child schemas to list.
+ Empty lists the database's top-level schemas.
"""
request_id: RequestId
engine: str
database: str
+ schema_path: list[str] = msgspec.field(default_factory=list)
class ListDataSourceConnectionCommand(Command):
diff --git a/marimo/_server/models/models.py b/marimo/_server/models/models.py
index 0862124a8f..c69fe45557 100644
--- a/marimo/_server/models/models.py
+++ b/marimo/_server/models/models.py
@@ -103,6 +103,7 @@ class ListSQLTablesRequest(ListSQLTablesCommand, tag=False):
engine=self.engine,
database=self.database,
schema=self.schema,
+ schema_path=self.schema_path,
)
@@ -112,6 +113,7 @@ class ListSQLSchemasRequest(ListSQLSchemasCommand, tag=False):
request_id=self.request_id,
engine=self.engine,
database=self.database,
+ schema_path=self.schema_path,
)
@@ -134,6 +136,7 @@ class PreviewSQLTableRequest(PreviewSQLTableCommand, tag=False):
database=self.database,
schema=self.schema,
table_name=self.table_name,
+ schema_path=self.schema_path,
)
diff --git a/marimo/_sql/connection_utils.py b/marimo/_sql/connection_utils.py
index 7b53d042eb..d9b8f14cf5 100644
--- a/marimo/_sql/connection_utils.py
+++ b/marimo/_sql/connection_utils.py
@@ -1,10 +1,57 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-from marimo._data.models import DataSourceConnection, DataTable, Schema
+from marimo._data.models import (
+ Database,
+ DataSourceConnection,
+ DataTable,
+ Schema,
+)
from marimo._messaging.notification import SQLDatabaseMetadata, SQLMetadata
+def _find_database(
+ connections: list[DataSourceConnection],
+ connection_name: str,
+ database_name: str,
+) -> Database | None:
+ for connection in connections:
+ if connection.name != connection_name:
+ continue
+ for database in connection.databases:
+ if database.name == database_name:
+ return database
+ return None
+
+
+def _find_schema_by_path(
+ schemas: list[Schema], path: list[str]
+) -> Schema | None:
+ """Descend `path` (segment names) into nested schema lists."""
+ if not path:
+ return None
+ head, *rest = path
+ for schema in schemas:
+ if schema.name == head:
+ if not rest:
+ return schema
+ return _find_schema_by_path(schema.child_schemas, rest)
+ return None
+
+
+def _find_table_schema(
+ database: Database, sql_metadata: SQLMetadata
+) -> Schema | None:
+ """Locate the schema holding a table. For nested schemas it is found by
+ descending `schema_path`; otherwise by schema name."""
+ if sql_metadata.schema_path:
+ return _find_schema_by_path(database.schemas, sql_metadata.schema_path)
+ for schema in database.schemas:
+ if schema.name == sql_metadata.schema:
+ return schema
+ return None
+
+
def update_table_in_connection(
connections: list[DataSourceConnection],
sql_metadata: SQLMetadata,
@@ -17,22 +64,18 @@ def update_table_in_connection(
sql_metadata: SQL metadata containing connection, database, schema info
updated_table: The updated table to replace the existing one
"""
- for connection in connections:
- if connection.name != sql_metadata.connection:
- continue
-
- for database in connection.databases:
- if database.name != sql_metadata.database:
- continue
-
- for schema in database.schemas:
- if schema.name != sql_metadata.schema:
- continue
-
- for i, table in enumerate(schema.tables):
- if table.name == updated_table.name:
- schema.tables[i] = updated_table
- return
+ database = _find_database(
+ connections, sql_metadata.connection, sql_metadata.database
+ )
+ if database is None:
+ return
+ schema = _find_table_schema(database, sql_metadata)
+ if schema is None:
+ return
+ for i, table in enumerate(schema.tables):
+ if table.name == updated_table.name:
+ schema.tables[i] = updated_table
+ return
def update_schema_list_in_connection(
@@ -42,22 +85,30 @@ def update_schema_list_in_connection(
) -> None:
"""Update a list of schemas in the connection hierarchy, updates in-place.
+ When `schema_path` is empty the database's top-level schemas are replaced;
+ otherwise the child schemas of the schema at that path are replaced.
+
Args:
connections: List of data source connections
sql_db_metadata: SQL database metadata containing connection, database info
updated_schema_list: The updated list of schemas to replace the existing ones
"""
- for connection in connections:
- if connection.name != sql_db_metadata.connection:
- continue
-
- for database in connection.databases:
- if database.name != sql_db_metadata.database:
- continue
-
- database.schemas = updated_schema_list
- database.schemas_resolved = True
+ database = _find_database(
+ connections, sql_db_metadata.connection, sql_db_metadata.database
+ )
+ if database is None:
+ return
+ if sql_db_metadata.schema_path:
+ parent = _find_schema_by_path(
+ database.schemas, sql_db_metadata.schema_path
+ )
+ if parent is None:
return
+ parent.child_schemas = updated_schema_list
+ parent.child_schemas_resolved = True
+ return
+ database.schemas = updated_schema_list
+ database.schemas_resolved = True
def update_table_list_in_connection(
@@ -72,18 +123,13 @@ def update_table_list_in_connection(
sql_metadata: SQL metadata containing connection, database, schema info
updated_table_list: The updated list of tables to replace the existing ones
"""
- for connection in connections:
- if connection.name != sql_metadata.connection:
- continue
-
- for database in connection.databases:
- if database.name != sql_metadata.database:
- continue
-
- for schema in database.schemas:
- if schema.name != sql_metadata.schema:
- continue
-
- schema.tables = updated_table_list
- schema.tables_resolved = True
- return
+ database = _find_database(
+ connections, sql_metadata.connection, sql_metadata.database
+ )
+ if database is None:
+ return
+ schema = _find_table_schema(database, sql_metadata)
+ if schema is None:
+ return
+ schema.tables = updated_table_list
+ schema.tables_resolved = True
diff --git a/marimo/_sql/engines/adbc.py b/marimo/_sql/engines/adbc.py
index de4b3bcd87..7133f6e344 100644
--- a/marimo/_sql/engines/adbc.py
+++ b/marimo/_sql/engines/adbc.py
@@ -309,8 +309,14 @@ class AdbcConnectionCatalog:
return databases
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
+ del schema_path # ADBC schemas don't nest
tables: list[DataTable] = []
objects_pylist = (
self._adbc_connection.adbc_get_objects(
@@ -362,9 +368,15 @@ class AdbcConnectionCatalog:
return tables
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
_ = database_name
+ del schema_path # ADBC schemas don't nest
try:
schema = self._adbc_connection.adbc_get_table_schema(
table_name, db_schema_filter=schema_name or None
@@ -509,9 +521,15 @@ class AdbcDBAPIEngine(SQLConnection[AdbcDbApiConnection]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _ = database, include_tables, include_table_details
+ _, _, _, _ = (
+ database,
+ include_tables,
+ include_table_details,
+ schema_path,
+ )
return []
def get_databases(
@@ -528,21 +546,33 @@ class AdbcDBAPIEngine(SQLConnection[AdbcDbApiConnection]):
)
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
return self._catalog.get_tables_in_schema(
schema=schema,
database=database,
include_table_details=include_table_details,
+ schema_path=schema_path,
)
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
return self._catalog.get_table_details(
table_name=table_name,
schema_name=schema_name,
database_name=database_name,
+ schema_path=schema_path,
)
def execute(
diff --git a/marimo/_sql/engines/clickhouse.py b/marimo/_sql/engines/clickhouse.py
index 259b8b328d..4a80097138 100644
--- a/marimo/_sql/engines/clickhouse.py
+++ b/marimo/_sql/engines/clickhouse.py
@@ -146,9 +146,15 @@ class ClickhouseEmbedded(SQLConnection[Optional["ChdbConnection"]]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _ = database, include_tables, include_table_details
+ _, _, _, _ = (
+ database,
+ include_tables,
+ include_table_details,
+ schema_path,
+ )
return []
# TODO: Implement the following functionalities
@@ -163,17 +169,27 @@ class ClickhouseEmbedded(SQLConnection[Optional["ChdbConnection"]]):
return []
def get_tables_in_schema(
- self, *, database: str, schema: str, include_table_details: bool
+ self,
+ *,
+ database: str,
+ schema: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Return all tables in a schema."""
- _, _, _ = database, schema, include_table_details
+ _, _, _, _ = database, schema, include_table_details, schema_path
return []
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get a single table from the engine."""
- _, _, _ = table_name, schema_name, database_name
+ _, _, _, _ = table_name, schema_name, database_name, schema_path
return None
def get_default_database(self) -> str | None:
@@ -280,9 +296,15 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _ = database, include_tables, include_table_details
+ _, _, _, _ = (
+ database,
+ include_tables,
+ include_table_details,
+ schema_path,
+ )
return []
def get_databases(
@@ -381,6 +403,7 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
schema: str,
database: str,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""
Return all tables in a given ClickHouse database.
@@ -389,10 +412,12 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
schema: The schema name. (ignored for ClickHouse)
database: The name of the database.
include_table_details: Whether to retrieve detailed table metadata.
+ schema_path: Unused; ClickHouse schemas don't nest.
Returns:
List of DataTable objects.
"""
+ del schema_path
tables, _ = self._get_tables_in_schema_with_resolution(
schema=schema,
database=database,
@@ -466,7 +491,12 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
return tables, len(tables) == len(table_names)
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""
Get detailed metadata for a given table in a database.
@@ -475,12 +505,14 @@ class ClickhouseServer(SQLConnection[Optional["ClickhouseClient"]]):
database_name: The database name.
schema_name: The schema name. (ignored for ClickHouse)
table_name: The table name.
+ schema_path: Unused; ClickHouse schemas don't nest.
Returns:
A DataTable object with detailed metadata,
or None if the table cannot be described.
"""
_ = schema_name
+ del schema_path
if self._connection is None:
return None
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.py
index 7a05949e6d..84f41f2c84 100644
--- a/marimo/_sql/engines/duckdb.py
+++ b/marimo/_sql/engines/duckdb.py
@@ -184,21 +184,37 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _ = database, include_tables, include_table_details
+ _, _, _, _ = (
+ database,
+ include_tables,
+ include_table_details,
+ schema_path,
+ )
return []
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Return all tables in a schema. This is currently implemented in get_databases_from_duckdb."""
- _, _, _ = database, schema, include_table_details
+ _, _, _, _ = database, schema, include_table_details, schema_path
return []
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get a single table from the engine. This is currently implemented in get_databases_from_duckdb."""
- _, _, _ = table_name, schema_name, database_name
+ _, _, _, _ = table_name, schema_name, database_name, schema_path
return None
diff --git a/marimo/_sql/engines/ibis.py b/marimo/_sql/engines/ibis.py
index 32ef77fbf8..bb5c77fdbc 100644
--- a/marimo/_sql/engines/ibis.py
+++ b/marimo/_sql/engines/ibis.py
@@ -199,8 +199,11 @@ class IbisEngine(SQLConnection["SQLBackend"]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
+ if schema_path:
+ return [] # Ibis backends don't expose nested schemas
meta_schemas = self._get_meta_schemas()
schemas: list[Schema] = []
@@ -260,9 +263,15 @@ class IbisEngine(SQLConnection["SQLBackend"]):
return ["information_schema", "pg_catalog"]
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Return all tables in a schema."""
+ del schema_path # Ibis backends don't expose nested schemas
if self._connection is None:
return []
@@ -364,9 +373,15 @@ class IbisEngine(SQLConnection["SQLBackend"]):
return list(set(table_names))
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get a single table from the engine."""
+ del schema_path # Ibis backends don't expose nested schemas
if self._connection is None:
return None
diff --git a/marimo/_sql/engines/pyiceberg.py b/marimo/_sql/engines/pyiceberg.py
index e1de4646f1..adc36595fc 100644
--- a/marimo/_sql/engines/pyiceberg.py
+++ b/marimo/_sql/engines/pyiceberg.py
@@ -54,9 +54,11 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
@property
def inference_config(self) -> InferenceConfig:
+ # List the first level of namespaces eagerly; fetch their tables and
+ # deeper sub-namespaces on expand.
return InferenceConfig(
auto_discover_schemas=True,
- auto_discover_tables="auto",
+ auto_discover_tables=False,
auto_discover_columns=False,
)
@@ -66,19 +68,6 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
def get_default_schema(self) -> str | None:
return None # Iceberg doesn't have schemas in the traditional sense
- # TODO: The following methods are currently not implemented.
- # We should consider implementing these in the future for better performance when users don't want to fetch everything.
- def get_schemas(
- self,
- *,
- database: str | None,
- include_tables: bool,
- include_table_details: bool,
- ) -> list[Schema]:
- """Get all schemas and optionally their tables. Keys are schema names."""
- _, _, _ = database, include_tables, include_table_details
- return []
-
def get_databases(
self,
*,
@@ -86,38 +75,38 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
include_tables: bool | Literal["auto"],
include_table_details: bool | Literal["auto"],
) -> list[Database]:
- """Get all databases from the engine."""
+ """Get all databases from the engine.
+
+ Each top-level Iceberg namespace becomes a `Database`. Nested
+ sub-namespaces are exposed as recursive child `Schema`s (see
+ `get_schemas`).
+ """
from pyiceberg.catalog import Catalog
- del include_schemas
+ schemas_resolved = self._resolve_should_auto_discover(include_schemas)
databases: list[Database] = []
- tables_resolved = self._resolve_should_auto_discover(include_tables)
try:
- namespaces = sorted(
- self._connection.list_namespaces()
- ) # Sort for consistent ordering
+ # Top-level namespaces only; children are discovered lazily.
+ namespaces = sorted(self._connection.list_namespaces())
for namespace in namespaces:
- tables = []
- if tables_resolved:
- tables = self.get_tables_in_schema(
- schema=NO_SCHEMA_NAME,
- database=Catalog.identifier_to_database(namespace),
+ database_name = Catalog.namespace_to_string(namespace)
+ schemas: list[Schema] = []
+ if schemas_resolved:
+ schemas = self.get_schemas(
+ database=database_name,
+ include_tables=self._resolve_should_auto_discover(
+ include_tables
+ ),
include_table_details=self._resolve_should_auto_discover(
include_table_details
),
)
-
databases.append(
Database(
- name=Catalog.identifier_to_database(namespace),
+ name=database_name,
dialect=self.dialect,
- schemas=[
- Schema(
- name=NO_SCHEMA_NAME,
- tables=tables,
- tables_resolved=tables_resolved,
- )
- ],
+ schemas=schemas,
+ schemas_resolved=schemas_resolved,
engine=self._engine_name,
)
)
@@ -134,16 +123,145 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
return databases
+ def get_schemas(
+ self,
+ *,
+ database: str | None,
+ include_tables: bool,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
+ ) -> list[Schema]:
+ """Get the schemas within a top-level namespace `database`.
+
+ Empty `schema_path` returns a schemaless `Schema` (the namespace's own
+ tables) plus one `Schema` per immediate sub-namespace; a non-empty path
+ returns the immediate sub-namespaces at that path.
+ """
+ if database is None:
+ raise ValueError("database is required for Iceberg schemas")
+
+ if schema_path:
+ return self._child_schemas(
+ (database, *schema_path),
+ include_tables=include_tables,
+ include_table_details=include_table_details,
+ )
+
+ # The database's sub-namespaces are siblings of (not nested under) the
+ # schemaless entry, so its own child_schemas are trivially resolved.
+ schemas: list[Schema] = [
+ Schema(
+ name=NO_SCHEMA_NAME,
+ tables=self.get_tables_in_schema(
+ schema=NO_SCHEMA_NAME,
+ database=database,
+ include_table_details=include_table_details,
+ )
+ if include_tables
+ else [],
+ tables_resolved=include_tables,
+ child_schemas=[],
+ child_schemas_resolved=True,
+ )
+ ]
+ schemas.extend(
+ self._child_schemas(
+ (database,),
+ include_tables=include_tables,
+ include_table_details=include_table_details,
+ )
+ )
+ return schemas
+
+ def _child_schemas(
+ self,
+ namespace: tuple[str, ...],
+ *,
+ include_tables: bool,
+ include_table_details: bool,
+ ) -> list[Schema]:
+ """Immediate child namespaces of `namespace` as Schemas."""
+ try:
+ children = sorted(self._connection.list_namespaces(namespace))
+ except Exception:
+ LOGGER.warning("Failed to list child namespaces", exc_info=True)
+ return []
+ return [
+ self._namespace_to_schema(
+ child,
+ include_tables=include_tables,
+ include_table_details=include_table_details,
+ )
+ for child in children
+ ]
+
+ def _namespace_to_schema(
+ self,
+ namespace: tuple[str, ...],
+ *,
+ include_tables: bool,
+ include_table_details: bool,
+ ) -> Schema:
+ """Convert an absolute namespace tuple into a Schema.
+
+ `include_tables` also gates sub-namespace recursion: when False, tables
+ and children are left deferred and fetched lazily on expand, so a
+ collapsed node does no catalog I/O.
+ """
+ from pyiceberg.catalog import Catalog
+
+ tables = (
+ self.get_tables_in_schema(
+ schema=NO_SCHEMA_NAME,
+ database=Catalog.namespace_to_string(namespace),
+ include_table_details=include_table_details,
+ )
+ if include_tables
+ else []
+ )
+ child_schemas = (
+ self._child_schemas(
+ namespace,
+ include_tables=include_tables,
+ include_table_details=include_table_details,
+ )
+ if include_tables
+ else []
+ )
+ return Schema(
+ name=namespace[-1],
+ tables=tables,
+ tables_resolved=include_tables,
+ child_schemas=child_schemas,
+ child_schemas_resolved=include_tables,
+ )
+
+ @staticmethod
+ def _qualified_namespace(
+ database: str, schema_path: list[str] | None
+ ) -> str:
+ """Fold a `database` + nested `schema_path` into a dotted namespace
+ string (the form pyiceberg's catalog calls expect)."""
+ if not schema_path:
+ return database
+ return ".".join([database, *schema_path])
+
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Return all tables in a schema."""
del schema # Not used since Iceberg doesn't have schemas
from pyiceberg.catalog import Catalog
+ namespace = self._qualified_namespace(database, schema_path)
try:
- tables = self._connection.list_tables(database)
+ tables = self._connection.list_tables(namespace)
if not include_table_details:
return [
DataTable(
@@ -167,7 +285,7 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
table: DataTable | None = self.get_table_details(
table_name=Catalog.table_name_from(table_name),
schema_name=NO_SCHEMA_NAME,
- database_name=Catalog.identifier_to_database(database),
+ database_name=namespace,
)
if table is not None:
data_tables.append(table)
@@ -178,10 +296,16 @@ class PyIcebergEngine(EngineCatalog["Catalog"]):
return []
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get a single table from the engine."""
del schema_name # Not used since Iceberg doesn't have schemas
+ database_name = self._qualified_namespace(database_name, schema_path)
try:
table = self._connection.load_table((database_name, table_name))
schema = table.schema()
diff --git a/marimo/_sql/engines/redshift.py b/marimo/_sql/engines/redshift.py
index 36a8b42eb4..c630c78220 100644
--- a/marimo/_sql/engines/redshift.py
+++ b/marimo/_sql/engines/redshift.py
@@ -222,8 +222,11 @@ class RedshiftEngine(SQLConnection["Connection"]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get schemas from the engine."""
+ if schema_path:
+ return [] # Redshift schemas don't nest
# Redshift doesn't use the concept of databases, just catalogs and schemas.
catalog = database
@@ -262,9 +265,15 @@ class RedshiftEngine(SQLConnection["Connection"]):
return output_schemas
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Get tables from the engine. Databases are treated as catalogs."""
+ del schema_path # Redshift schemas don't nest
output_tables: list[DataTable] = []
@@ -335,9 +344,15 @@ class RedshiftEngine(SQLConnection["Connection"]):
return columns
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get detailed metadata for a given table in a database."""
+ del schema_path # Redshift schemas don't nest
with self._connection.cursor() as cursor:
try:
diff --git a/marimo/_sql/engines/sqlalchemy.py b/marimo/_sql/engines/sqlalchemy.py
index a69159349f..d8f3af67c9 100644
--- a/marimo/_sql/engines/sqlalchemy.py
+++ b/marimo/_sql/engines/sqlalchemy.py
@@ -456,8 +456,11 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
"""Get all schemas and optionally their tables. Keys are schema names."""
+ if schema_path:
+ return [] # SQLAlchemy schemas don't nest
if database is None:
schema_names: list[str] = []
@@ -520,9 +523,15 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
), inspector.get_view_names(schema=schema)
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
"""Return all tables in a schema."""
+ del schema_path # SQLAlchemy schemas don't nest
table_names, view_names = self._get_table_names(
schema=schema, database=database
@@ -618,9 +627,15 @@ class SQLAlchemyEngine(SQLConnection["Engine"]):
return index_columns
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
"""Get a single table from the engine."""
+ del schema_path # SQLAlchemy schemas don't nest
columns = self._get_columns(
table_name, schema=schema_name, database=database_name
diff --git a/marimo/_sql/engines/types.py b/marimo/_sql/engines/types.py
index 0d5e3833cb..a2b493f034 100644
--- a/marimo/_sql/engines/types.py
+++ b/marimo/_sql/engines/types.py
@@ -120,20 +120,44 @@ class EngineCatalog(BaseEngine[CONN], ABC):
database: str | None,
include_tables: bool,
include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[Schema]:
- """Return the schemas for a database in the engine."""
+ """Return schemas within a database.
+
+ Empty `schema_path` lists the database's top-level schemas; a non-empty
+ path lists the child schemas at that path. Only nested-namespace engines
+ (e.g. Iceberg) honour a non-empty path; flat engines return `[]` for one.
+ """
@abstractmethod
def get_tables_in_schema(
- self, *, schema: str, database: str, include_table_details: bool
+ self,
+ *,
+ schema: str,
+ database: str,
+ include_table_details: bool,
+ schema_path: list[str] | None = None,
) -> list[DataTable]:
- """Return all tables in a schema."""
+ """Return all tables in a schema.
+
+ Nested-namespace engines locate the schema via `schema_path` (relative
+ to `database`); flat engines use `schema` and ignore it.
+ """
@abstractmethod
def get_table_details(
- self, *, table_name: str, schema_name: str, database_name: str
+ self,
+ *,
+ table_name: str,
+ schema_name: str,
+ database_name: str,
+ schema_path: list[str] | None = None,
) -> DataTable | None:
- """Get a single table from the engine."""
+ """Get a single table from the engine.
+
+ Nested-namespace engines locate the table via `schema_path` (relative to
+ `database_name`); flat engines ignore it.
+ """
def _resolve_should_auto_discover(
self, value: bool | Literal["auto"]
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 5dc322bb37..e95b64a7c3 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -2903,7 +2903,9 @@ components:
\ schemas in a database. Used by the SQL editor for\n schema selection.\n\
\n Attributes:\n request_id: Unique identifier for this request.\n\
\ engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).\n \
- \ database: Database to query."
+ \ database: Database to query.\n schema_path: Parent schema path\
+ \ whose child schemas to list.\n Empty lists the database's top-level\
+ \ schemas."
properties:
database:
type: string
@@ -2911,6 +2913,11 @@ components:
type: string
requestId:
$ref: '#/components/schemas/RequestId'
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
type:
enum:
- list-sql-schemas
@@ -2929,6 +2936,11 @@ components:
type: string
requestId:
$ref: '#/components/schemas/RequestId'
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
required:
- requestId
- engine
@@ -2941,7 +2953,8 @@ components:
\n Attributes:\n request_id: Unique identifier for this request.\n\
\ engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).\n \
\ database: Database to query.\n schema: Schema to list tables\
- \ from."
+ \ from.\n schema_path: Path of nested schemas (relative to `database`)\
+ \ for\n catalogs with nested schemas. Empty for the top level."
properties:
database:
type: string
@@ -2951,6 +2964,11 @@ components:
$ref: '#/components/schemas/RequestId'
schema:
type: string
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
type:
enum:
- list-sql-tables
@@ -2972,6 +2990,11 @@ components:
$ref: '#/components/schemas/RequestId'
schema:
type: string
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
required:
- requestId
- engine
@@ -3933,7 +3956,9 @@ components:
\ Attributes:\n request_id: Unique identifier for this request.\n \
\ engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).\n \
\ database: Database containing the table.\n schema: Schema containing\
- \ the table.\n table_name: Table to preview."
+ \ the table.\n table_name: Table to preview.\n schema_path:\
+ \ Path of nested schemas (relative to `database`) for\n catalogs\
+ \ with nested schemas. Empty for the top level."
properties:
database:
type: string
@@ -3943,6 +3968,11 @@ components:
$ref: '#/components/schemas/RequestId'
schema:
type: string
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
tableName:
type: string
type:
@@ -3967,6 +3997,11 @@ components:
$ref: '#/components/schemas/RequestId'
schema:
type: string
+ schemaPath:
+ default: []
+ items:
+ type: string
+ type: array
tableName:
type: string
required:
@@ -4310,12 +4345,19 @@ components:
type: object
SQLDatabaseMetadata:
description: "SQL database metadata.\n\n Attributes:\n connection:\
- \ Connection identifier.\n database: Database name."
+ \ Connection identifier.\n database: Database name.\n schema_path:\
+ \ Parent schema path the schemas belong under. Empty for\n the\
+ \ database's top level."
properties:
connection:
type: string
database:
type: string
+ schema_path:
+ default: []
+ items:
+ type: string
+ type: array
required:
- connection
- database
@@ -4324,7 +4366,8 @@ components:
SQLMetadata:
description: "SQL database and schema metadata.\n\n Attributes:\n \
\ connection: Connection identifier.\n database: Database name.\n \
- \ schema: Schema name."
+ \ schema: Schema name.\n schema_path: Path of nested schemas\
+ \ (relative to `database`). Empty\n for the top level."
properties:
connection:
type: string
@@ -4332,6 +4375,11 @@ components:
type: string
schema:
type: string
+ schema_path:
+ default: []
+ items:
+ type: string
+ type: array
type:
enum:
- sql-metadata
@@ -4501,12 +4549,24 @@ components:
title: SaveUserConfigurationRequest
type: object
Schema:
- description: "Represents a database schema and its tables.\n\nAttributes:\n\
- \ name (str): The name of the schema.\n tables (List[DataTable]): Tables\
- \ in this schema.\n tables_resolved (bool): True when `tables` has been\
- \ enumerated\n False when table discovery was deferred. Defaults to\
- \ True"
- properties:
+ description: "Represents a database schema and its tables.\n\nA schema may itself\
+ \ contain nested child schemas, e.g. for catalogs with\nhierarchical namespaces\
+ \ such as Iceberg (`top.nested.deep`).\n\nAttributes:\n name (str): The\
+ \ name of the schema.\n tables (List[DataTable]): Tables in this schema.\n\
+ \ tables_resolved (bool): True when `tables` has been enumerated\n \
+ \ False when table discovery was deferred. Defaults to True\n child_schemas\
+ \ (List[Schema]): Nested child schemas (sub-namespaces).\n child_schemas_resolved\
+ \ (bool): True when `child_schemas` has been\n enumerated. False when\
+ \ discovery was deferred. Defaults to True"
+ properties:
+ child_schemas:
+ default: []
+ items:
+ $ref: '#/components/schemas/Schema'
+ type: array
+ child_schemas_resolved:
+ default: true
+ type: boolean
name:
type: string
tables:
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index befb476eba..71332372e6 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -5175,11 +5175,15 @@ export interface components {
* request_id: Unique identifier for this request.
* engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).
* database: Database to query.
+ * schema_path: Parent schema path whose child schemas to list.
+ * Empty lists the database's top-level schemas.
*/
ListSQLSchemasCommand: {
database: string;
engine: string;
requestId: components["schemas"]["RequestId"];
+ /** @default [] */
+ schemaPath?: string[];
/** @enum {unknown} */
type: "list-sql-schemas";
};
@@ -5188,6 +5192,8 @@ export interface components {
database: string;
engine: string;
requestId: components["schemas"]["RequestId"];
+ /** @default [] */
+ schemaPath?: string[];
};
/**
* ListSQLTablesCommand
@@ -5201,12 +5207,16 @@ export interface components {
* engine: SQL engine ('postgresql', 'mysql', 'duckdb', etc.).
* database: Database to query.
* schema: Schema to list tables from.
+ * schema_path: Path of nested schemas (relative to `database`) for
+ * catalogs with nested schemas. Empty for the top level.
*/
ListSQLTablesCommand: {
database: string;
engine: string;
requestId: components["schemas"]["RequestId"];
schema: string;
+ /** @default [] */
+ schemaPath?: string[];
/** @enum {unknown} */
type: "list-sql-tables";
};
@@ -5216,6 +5226,8 @@ export interface components {
engine: string;
requestId: components["schemas"]["RequestId"];
schema: string;
+ /** @default [] */
+ schemaPath?: string[];
};
/**
* ListSecretKeysCommand
@@ -5769,12 +5781,16 @@ export interface components {
* database: Database containing the table.
* schema: Schema containing the table.
* table_name: Table to preview.
+ * schema_path: Path of nested schemas (relative to `database`) for
+ * catalogs with nested schemas. Empty for the top level.
*/
PreviewSQLTableCommand: {
database: string;
engine: string;
requestId: components["schemas"]["RequestId"];
schema: string;
+ /** @default [] */
+ schemaPath?: string[];
tableName: string;
/** @enum {unknown} */
type: "preview-sql-table";
@@ -5785,6 +5801,8 @@ export interface components {
engine: string;
requestId: components["schemas"]["RequestId"];
schema: string;
+ /** @default [] */
+ schemaPath?: string[];
tableName: string;
};
/**
@@ -6031,10 +6049,14 @@ export interface components {
* Attributes:
* connection: Connection identifier.
* database: Database name.
+ * schema_path: Parent schema path the schemas belong under. Empty for
+ * the database's top level.
*/
SQLDatabaseMetadata: {
connection: string;
database: string;
+ /** @default [] */
+ schema_path?: string[];
};
/**
* SQLMetadata
@@ -6044,11 +6066,15 @@ export interface components {
* connection: Connection identifier.
* database: Database name.
* schema: Schema name.
+ * schema_path: Path of nested schemas (relative to `database`). Empty
+ * for the top level.
*/
SQLMetadata: {
connection: string;
database: string;
schema: string;
+ /** @default [] */
+ schema_path?: string[];
/** @enum {unknown} */
type: "sql-metadata";
};
@@ -6151,13 +6177,23 @@ export interface components {
* Schema
* @description Represents a database schema and its tables.
*
+ * A schema may itself contain nested child schemas, e.g. for catalogs with
+ * hierarchical namespaces such as Iceberg (`top.nested.deep`).
+ *
* Attributes:
* name (str): The name of the schema.
* tables (List[DataTable]): Tables in this schema.
* tables_resolved (bool): True when `tables` has been enumerated
* False when table discovery was deferred. Defaults to True
+ * child_schemas (List[Schema]): Nested child schemas (sub-namespaces).
+ * child_schemas_resolved (bool): True when `child_schemas` has been
+ * enumerated. False when discovery was deferred. Defaults to True
*/
Schema: {
+ /** @default [] */
+ child_schemas?: components["schemas"]["Schema"][];
+ /** @default true */
+ child_schemas_resolved?: boolean;
name: string;
tables: components["schemas"]["DataTable"][];
/** @default true */
diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.py
index 9df5814c95..720d0e057c 100644
--- a/tests/_runtime/test_runtime_datasets.py
+++ b/tests/_runtime/test_runtime_datasets.py
@@ -281,6 +281,45 @@ class TestPreviewSQLSchemaList:
)
]
+ async def test_nested_schema_path_echoed(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecuteCellCommand],
+ ) -> None:
+ """A request with a schema_path lists the child schemas at that path
+ and echoes the path in the response metadata. Catalog engines without
+ hierarchical namespaces return an empty list."""
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_sql_schema_list_request = ListSQLSchemasCommand(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ database="test",
+ schema_path=["sub"],
+ )
+ await k.handle_message(preview_sql_schema_list_request)
+
+ results = [
+ op
+ for op in stream.operations
+ if isinstance(op, SQLSchemaListPreviewNotification)
+ ]
+ assert results == [
+ SQLSchemaListPreviewNotification(
+ request_id=RequestId("0"),
+ schemas=[],
+ error=None,
+ metadata=SQLDatabaseMetadata(
+ connection=DUCKDB_CONN,
+ database="test",
+ schema_path=["sub"],
+ ),
+ )
+ ]
+
@pytest.mark.skipif(not HAS_SQL, reason="SQL deps not available")
class TestPreviewSQLTableList:
diff --git a/tests/_server/test_sql_request_models.py b/tests/_server/test_sql_request_models.py
new file mode 100644
index 0000000000..653d81040e
--- /dev/null
+++ b/tests/_server/test_sql_request_models.py
@@ -0,0 +1,50 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import msgspec
+
+from marimo._server.models.models import (
+ ListSQLSchemasRequest,
+ ListSQLTablesRequest,
+ PreviewSQLTableRequest,
+)
+
+
+def test_list_sql_schemas_request_preserves_schema_path() -> None:
+ # Wire format is camelCase (Command uses rename="camel").
+ body = (
+ b'{"requestId":"1","engine":"e","database":"top",'
+ b'"schemaPath":["nested"]}'
+ )
+ request = msgspec.json.decode(body, type=ListSQLSchemasRequest)
+ assert request.schema_path == ["nested"]
+ # as_command must not drop the field.
+ assert request.as_command().schema_path == ["nested"]
+
+
+def test_list_sql_tables_request_preserves_schema_path() -> None:
+ body = (
+ b'{"requestId":"1","engine":"e","database":"top","schema":"deep",'
+ b'"schemaPath":["nested","deep"]}'
+ )
+ request = msgspec.json.decode(body, type=ListSQLTablesRequest)
+ assert request.schema_path == ["nested", "deep"]
+ assert request.as_command().schema_path == ["nested", "deep"]
+
+
+def test_preview_sql_table_request_preserves_schema_path() -> None:
+ body = (
+ b'{"requestId":"1","engine":"e","database":"top","schema":"deep",'
+ b'"tableName":"t","schemaPath":["nested","deep"]}'
+ )
+ request = msgspec.json.decode(body, type=PreviewSQLTableRequest)
+ assert request.schema_path == ["nested", "deep"]
+ assert request.as_command().schema_path == ["nested", "deep"]
+
+
+def test_schema_path_defaults_to_empty() -> None:
+ # Older clients omit the field entirely.
+ body = b'{"requestId":"1","engine":"e","database":"top","schema":"s","tableName":"t"}'
+ request = msgspec.json.decode(body, type=PreviewSQLTableRequest)
+ assert request.schema_path == []
+ assert request.as_command().schema_path == []
diff --git a/tests/_sql/test_connection_utils.py b/tests/_sql/test_connection_utils.py
index 1ae69b97fe..33931d9146 100644
--- a/tests/_sql/test_connection_utils.py
+++ b/tests/_sql/test_connection_utils.py
@@ -15,6 +15,7 @@ from marimo._data.models import (
)
from marimo._messaging.notification import SQLDatabaseMetadata, SQLMetadata
from marimo._sql.connection_utils import (
+ _find_schema_by_path,
update_schema_list_in_connection,
update_table_in_connection,
update_table_list_in_connection,
@@ -578,3 +579,125 @@ class TestPerformance:
print("cProfile Results (top 20):")
print("=" * 80)
stats.print_stats(20)
+
+
+def _create_nested_connection() -> list[DataSourceConnection]:
+ """Connection with a top-level namespace ("top") holding a recursive
+ sub-namespace tree: top -> nested -> deep."""
+ deep = Schema(
+ name="deep",
+ tables=[],
+ tables_resolved=False,
+ child_schemas=[],
+ child_schemas_resolved=False,
+ )
+ nested = Schema(
+ name="nested",
+ tables=[create_test_table("table4")],
+ tables_resolved=True,
+ child_schemas=[deep],
+ child_schemas_resolved=True,
+ )
+ top = Database(
+ name="top",
+ dialect="iceberg",
+ schemas=[
+ Schema(name="", tables=[], tables_resolved=True),
+ nested,
+ ],
+ )
+ return [
+ DataSourceConnection(
+ source="iceberg",
+ dialect="iceberg",
+ name="my_iceberg",
+ display_name="iceberg (my_iceberg)",
+ databases=[top],
+ )
+ ]
+
+
+class TestFindSchemaByPath:
+ def test_finds_top_level(self) -> None:
+ connections = _create_nested_connection()
+ schemas = connections[0].databases[0].schemas
+ found = _find_schema_by_path(schemas, ["nested"])
+ assert found is not None
+ assert found.name == "nested"
+
+ def test_descends_into_nested(self) -> None:
+ connections = _create_nested_connection()
+ schemas = connections[0].databases[0].schemas
+ found = _find_schema_by_path(schemas, ["nested", "deep"])
+ assert found is not None
+ assert found.name == "deep"
+
+ def test_missing_path_returns_none(self) -> None:
+ connections = _create_nested_connection()
+ schemas = connections[0].databases[0].schemas
+ assert _find_schema_by_path(schemas, ["nested", "missing"]) is None
+ assert _find_schema_by_path(schemas, []) is None
+
+
+class TestNestedNamespaceUpdates:
+ def test_update_child_schema_list_at_path(self) -> None:
+ """Resolving child namespaces of a nested namespace updates the right
+ node in-place."""
+ connections = _create_nested_connection()
+ sql_db_metadata = SQLDatabaseMetadata(
+ connection="my_iceberg",
+ database="top",
+ schema_path=["nested"],
+ )
+ new_children = [Schema(name="deep", tables=[])]
+
+ update_schema_list_in_connection(
+ connections, sql_db_metadata, new_children
+ )
+
+ nested = _find_schema_by_path(
+ connections[0].databases[0].schemas, ["nested"]
+ )
+ assert nested is not None
+ assert [s.name for s in nested.child_schemas] == ["deep"]
+ assert nested.child_schemas_resolved is True
+
+ def test_update_table_list_at_nested_path(self) -> None:
+ """Resolving tables of a deeply nested namespace targets that schema."""
+ connections = _create_nested_connection()
+ sql_metadata = SQLMetadata(
+ connection="my_iceberg",
+ database="top",
+ schema="deep",
+ schema_path=["nested", "deep"],
+ )
+ new_tables = [create_test_table("table5")]
+
+ update_table_list_in_connection(connections, sql_metadata, new_tables)
+
+ deep = _find_schema_by_path(
+ connections[0].databases[0].schemas, ["nested", "deep"]
+ )
+ assert deep is not None
+ assert [t.name for t in deep.tables] == ["table5"]
+ assert deep.tables_resolved is True
+
+ def test_update_table_at_nested_path(self) -> None:
+ """A single-table update descends the namespace path."""
+ connections = _create_nested_connection()
+ sql_metadata = SQLMetadata(
+ connection="my_iceberg",
+ database="top",
+ schema="nested",
+ schema_path=["nested"],
+ )
+ updated = create_test_table("table4")
+ updated.num_rows = 999
+
+ update_table_in_connection(connections, sql_metadata, updated)
+
+ nested = _find_schema_by_path(
+ connections[0].databases[0].schemas, ["nested"]
+ )
+ assert nested is not None
+ assert nested.tables[0].num_rows == 999
diff --git a/tests/_sql/test_engines.py b/tests/_sql/test_engines.py
index 48550213bb..b91aa260f8 100644
--- a/tests/_sql/test_engines.py
+++ b/tests/_sql/test_engines.py
@@ -483,3 +483,22 @@ def test_try_convert_to_polars() -> None:
assert df is None
assert isinstance(error, pl.exceptions.ComputeError)
assert str(error) == "Test error"
+
+
[email protected]("sqlalchemy")
+def test_flat_engine_get_schemas_ignores_nested_path() -> None:
+ """Flat engines have no nested schemas, so get_schemas returns [] for a
+ non-empty schema_path. Engines with nested namespaces (e.g. Iceberg) honour
+ it."""
+ import sqlalchemy as sa
+
+ engine = SQLAlchemyEngine(sa.create_engine("sqlite:///:memory:"))
+ assert (
+ engine.get_schemas(
+ database="main",
+ include_tables=False,
+ include_table_details=False,
+ schema_path=["nested"],
+ )
+ == []
+ )
diff --git a/tests/_sql/test_pyiceberg.py b/tests/_sql/test_pyiceberg.py
index 685a384d13..1a52de4cfa 100644
--- a/tests/_sql/test_pyiceberg.py
+++ b/tests/_sql/test_pyiceberg.py
@@ -44,9 +44,11 @@ def memory_catalog() -> Generator[Catalog, None, None]:
# Create mock catalog
catalog = InMemoryCatalog("test_catalog")
- # Create namespaces
+ # Create namespaces, including nested ones
catalog.create_namespace("default")
catalog.create_namespace("test_namespace")
+ catalog.create_namespace(("top", "nested"))
+ catalog.create_namespace(("top", "nested", "deep"))
# Create actual schema object with real fields
schema = IcebergSchema(
@@ -69,15 +71,21 @@ def memory_catalog() -> Generator[Catalog, None, None]:
catalog.create_table(("default", "table1"), schema)
catalog.create_table(("default", "table2"), schema)
catalog.create_table(("test_namespace", "table3"), schema)
+ catalog.create_table(("top", "nested", "table4"), schema)
+ catalog.create_table(("top", "nested", "deep", "table5"), schema)
assert len(catalog.list_tables("default")) == 2
assert len(catalog.list_tables("test_namespace")) == 1
+ assert len(catalog.list_tables(("top", "nested"))) == 1
+ assert len(catalog.list_tables(("top", "nested", "deep"))) == 1
yield catalog
catalog.drop_table(("default", "table1"))
catalog.drop_table(("default", "table2"))
catalog.drop_table(("test_namespace", "table3"))
+ catalog.drop_table(("top", "nested", "table4"))
+ catalog.drop_table(("top", "nested", "deep", "table5"))
def get_expected_table(
@@ -243,44 +251,219 @@ def test_pyiceberg_get_tables_in_schema(memory_catalog: Catalog) -> None:
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
def test_pyiceberg_get_databases(memory_catalog: Catalog) -> None:
- """Test PyIcebergEngine get_databases method."""
+ """Each top-level namespace is a Database; sub-namespaces are recursive
+ child Schemas."""
engine_name = VariableName("my_iceberg")
engine = PyIcebergEngine(memory_catalog, engine_name=engine_name)
- # Test with all parameters True
+ # Test with all parameters True (eager)
databases = engine.get_databases(
include_schemas=True, include_tables=True, include_table_details=True
)
assert isinstance(databases, list)
- assert len(databases) == 2
- assert databases[0].name == "default"
- assert databases[0].dialect == "iceberg"
- assert len(databases[0].schemas) == 1
- assert databases[0].schemas[0].name == NO_SCHEMA_NAME
- assert len(databases[0].schemas[0].tables) == 2
-
- assert databases[1].name == "test_namespace"
- assert len(databases[1].schemas) == 1
- assert databases[1].schemas[0].name == NO_SCHEMA_NAME
- assert len(databases[1].schemas[0].tables) == 1
-
- # Test with include_tables=False
+ # Only top-level namespaces become Databases.
+ by_name = {db.name: db for db in databases}
+ assert set(by_name) == {"default", "test_namespace", "top"}
+
+ # "default" has its own tables in a schemaless Schema, no sub-namespaces.
+ default_schemas = {s.name: s for s in by_name["default"].schemas}
+ assert by_name["default"].dialect == "iceberg"
+ assert set(default_schemas) == {NO_SCHEMA_NAME}
+ assert len(default_schemas[NO_SCHEMA_NAME].tables) == 2
+ assert default_schemas[NO_SCHEMA_NAME].child_schemas == []
+
+ assert {s.name for s in by_name["test_namespace"].schemas} == {
+ NO_SCHEMA_NAME
+ }
+
+ # "top" has no tables of its own but contains the "nested" sub-namespace.
+ top_schemas = {s.name: s for s in by_name["top"].schemas}
+ assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
+ assert top_schemas[NO_SCHEMA_NAME].tables == []
+
+ nested = top_schemas["nested"]
+ assert [t.name for t in nested.tables] == ["table4"]
+ # "nested" recursively contains "deep", which holds "table5".
+ deep_by_name = {s.name: s for s in nested.child_schemas}
+ assert set(deep_by_name) == {"deep"}
+ assert [t.name for t in deep_by_name["deep"].tables] == ["table5"]
+
+ # Test with include_tables=False (schemas listed, tables deferred)
databases = engine.get_databases(
include_schemas=True, include_tables=False, include_table_details=True
)
+ by_name = {db.name: db for db in databases}
+ assert set(by_name) == {"default", "test_namespace", "top"}
+ top_schemas = {s.name: s for s in by_name["top"].schemas}
+ # Sub-namespace is present but its tables/children are deferred.
+ assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
+ assert top_schemas["nested"].tables_resolved is False
+ assert top_schemas["nested"].child_schemas_resolved is False
+ assert top_schemas["nested"].tables == []
+ assert top_schemas["nested"].child_schemas == []
- assert isinstance(databases, list)
- assert len(databases) == 2
- assert databases[0].name == "default"
- assert len(databases[0].schemas) == 1
- assert databases[0].schemas[0].name == NO_SCHEMA_NAME
- assert len(databases[0].schemas[0].tables) == 0
- assert databases[1].name == "test_namespace"
- assert len(databases[1].schemas) == 1
- assert databases[1].schemas[0].name == NO_SCHEMA_NAME
- assert len(databases[1].schemas[0].tables) == 0
[email protected](not HAS_PYICEBERG, reason="PyIceberg not installed")
+def test_pyiceberg_get_databases_lazy_schemas(memory_catalog: Catalog) -> None:
+ """With include_schemas=False, databases defer schema discovery."""
+ engine = PyIcebergEngine(
+ memory_catalog, engine_name=VariableName("my_iceberg")
+ )
+
+ databases = engine.get_databases(
+ include_schemas=False,
+ include_tables=False,
+ include_table_details=False,
+ )
+ assert {db.name for db in databases} == {
+ "default",
+ "test_namespace",
+ "top",
+ }
+ for db in databases:
+ assert db.schemas_resolved is False
+ assert db.schemas == []
+
+
[email protected](not HAS_PYICEBERG, reason="PyIceberg not installed")
+def test_pyiceberg_connection_is_lazy(memory_catalog: Catalog) -> None:
+ """The initial connection lists top-level namespaces and the first level of
+ sub-namespaces, but defers their tables and deeper namespaces until expand."""
+ from marimo._sql.get_engines import engine_to_data_source_connection
+
+ engine = PyIcebergEngine(
+ memory_catalog, engine_name=VariableName("my_iceberg")
+ )
+ # The first level of schemas is eager; tables/columns are deferred.
+ assert engine.inference_config.auto_discover_schemas is True
+ assert engine.inference_config.auto_discover_tables is False
+ assert engine.inference_config.auto_discover_columns is False
+
+ connection = engine_to_data_source_connection(
+ VariableName("my_iceberg"), engine
+ )
+ by_name = {db.name: db for db in connection.databases}
+ assert set(by_name) == {"default", "test_namespace", "top"}
+
+ # First-level schemas are resolved...
+ top = by_name["top"]
+ assert top.schemas_resolved is True
+ top_schemas = {s.name: s for s in top.schemas}
+ assert set(top_schemas) == {NO_SCHEMA_NAME, "nested"}
+
+ # ...but their tables and deeper sub-namespaces are deferred.
+ assert top_schemas[NO_SCHEMA_NAME].tables_resolved is False
+ assert top_schemas[NO_SCHEMA_NAME].tables == []
+ assert top_schemas["nested"].tables_resolved is False
+ assert top_schemas["nested"].child_schemas_resolved is False
+ assert top_schemas["nested"].tables == []
+ assert top_schemas["nested"].child_schemas == []
+
+
[email protected](not HAS_PYICEBERG, reason="PyIceberg not installed")
+def test_pyiceberg_get_schemas_by_path(memory_catalog: Catalog) -> None:
+ """get_schemas lists one level at a time, selected by schema_path."""
+ engine = PyIcebergEngine(
+ memory_catalog, engine_name=VariableName("my_iceberg")
+ )
+
+ # Top level: the schemaless entry plus the immediate child "nested"
+ # (deferred, not recursed).
+ schemas = engine.get_schemas(
+ database="top",
+ include_tables=False,
+ include_table_details=False,
+ schema_path=[],
+ )
+ assert [s.name for s in schemas] == [NO_SCHEMA_NAME, "nested"]
+ nested = schemas[1]
+ assert nested.child_schemas_resolved is False
+ assert nested.tables_resolved is False
+ assert nested.child_schemas == []
+
+ # Immediate child of "top.nested" is "deep".
+ schemas = engine.get_schemas(
+ database="top",
+ include_tables=False,
+ include_table_details=False,
+ schema_path=["nested"],
+ )
+ assert [s.name for s in schemas] == ["deep"]
+
+ # "top.nested.deep" is a leaf.
+ assert (
+ engine.get_schemas(
+ database="top",
+ include_tables=False,
+ include_table_details=False,
+ schema_path=["nested", "deep"],
+ )
+ == []
+ )
+
+
[email protected](not HAS_PYICEBERG, reason="PyIceberg not installed")
+def test_pyiceberg_nested_namespace_tables(memory_catalog: Catalog) -> None:
+ """Tables of a nested namespace are reachable via its dotted name."""
+ engine = PyIcebergEngine(
+ memory_catalog, engine_name=VariableName("my_iceberg")
+ )
+
+ tables = engine.get_tables_in_schema(
+ schema=NO_SCHEMA_NAME,
+ database="top.nested",
+ include_table_details=True,
+ )
+ assert [t.name for t in tables] == ["table4"]
+ assert tables[0].num_columns == 3
+
+ table = engine.get_table_details(
+ table_name="table5",
+ schema_name=NO_SCHEMA_NAME,
+ database_name="top.nested.deep",
+ )
+ assert table is not None
+ assert table.name == "table5"
+ assert len(table.columns) == 3
+
+
[email protected](not HAS_PYICEBERG, reason="PyIceberg not installed")
+def test_pyiceberg_table_calls_fold_schema_path(
+ memory_catalog: Catalog,
+) -> None:
+ """The handler passes the top-level `database` plus a `schema_path`; the
+ engine folds them into a dotted namespace internally (this replaced the
+ handler-side `_table_database` helper)."""
+ engine = PyIcebergEngine(
+ memory_catalog, engine_name=VariableName("my_iceberg")
+ )
+
+ # database + schema_path is equivalent to the pre-folded dotted database.
+ tables = engine.get_tables_in_schema(
+ schema=NO_SCHEMA_NAME,
+ database="top",
+ schema_path=["nested"],
+ include_table_details=True,
+ )
+ assert [t.name for t in tables] == ["table4"]
+
+ table = engine.get_table_details(
+ table_name="table5",
+ schema_name=NO_SCHEMA_NAME,
+ database_name="top",
+ schema_path=["nested", "deep"],
+ )
+ assert table is not None
+ assert table.name == "table5"
+
+ # Empty / missing schema_path leaves the database untouched.
+ assert PyIcebergEngine._qualified_namespace("top", []) == "top"
+ assert PyIcebergEngine._qualified_namespace("top", None) == "top"
+ assert (
+ PyIcebergEngine._qualified_namespace("top", ["nested", "deep"])
+ == "top.nested.deep"
+ )
@pytest.mark.skipif(not HAS_PYICEBERG, reason="PyIceberg not installed")
@@ -297,9 +480,10 @@ def test_pyiceberg_auto_discovery(memory_catalog: Catalog) -> None:
)
assert isinstance(databases, list)
- assert len(databases) == 2
- assert databases[0].schemas[0].name == NO_SCHEMA_NAME
- assert len(databases[0].schemas[0].tables) == 2
+ assert len(databases) == 3
+ by_name = {db.name: db for db in databases}
+ default_schemas = {s.name: s for s in by_name["default"].schemas}
+ assert len(default_schemas[NO_SCHEMA_NAME].tables) == 2
# Test with _is_cheap_discovery mocked to return False
with mock.patch.object(
@@ -312,5 +496,7 @@ def test_pyiceberg_auto_discovery(memory_catalog: Catalog) -> None:
)
assert isinstance(databases, list)
- assert len(databases) == 2
- assert len(databases[0].schemas[0].tables) == 0
+ assert len(databases) == 3
+ for db in databases:
+ assert db.schemas_resolved is False
+ assert db.schemas == []