Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(datasources): only auto-expand search matches in the data sources tree (#9955)

Shahmir Varqhaea00c8519625
diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.ts
index 1d7483eb91..bad765cf5b 100644
--- a/frontend/src/components/datasources/__tests__/utils.test.ts
+++ b/frontend/src/components/datasources/__tests__/utils.test.ts
@@ -3,8 +3,18 @@
import { describe, expect, it } from "vitest";
import type { SQLTableContext } from "@/core/datasets/data-source-connections";
import { DUCKDB_ENGINE } from "@/core/datasets/engines";
-import type { DataTable, DataTableColumn } from "@/core/kernel/messages";
-import { sqlCode, tableUniqueId } from "../utils";
+import type {
+ Database,
+ DatabaseSchema,
+ DataTable,
+ DataTableColumn,
+} from "@/core/kernel/messages";
+import {
+ schemaSubtreeMatchesSearch,
+ shouldExpandDatabaseForSearch,
+ sqlCode,
+ tableUniqueId,
+} from "../utils";
describe("sqlCode", () => {
const mockTable: DataTable = {
@@ -519,3 +529,118 @@ describe("tableUniqueId", () => {
);
});
});
+
+function makeTable(name: string): DataTable {
+ return {
+ name,
+ columns: [],
+ source: "memory",
+ source_type: "local",
+ type: "table",
+ engine: null,
+ indexes: null,
+ num_columns: null,
+ num_rows: null,
+ variable_name: null,
+ primary_keys: null,
+ };
+}
+
+function makeSchema(opts: {
+ name: string;
+ tables: DataTable[];
+ tables_resolved?: boolean;
+ child_schemas?: DatabaseSchema[];
+ child_schemas_resolved?: boolean;
+}): DatabaseSchema {
+ return {
+ name: opts.name,
+ tables: opts.tables,
+ tables_resolved: opts.tables_resolved ?? true,
+ child_schemas: opts.child_schemas ?? [],
+ child_schemas_resolved: opts.child_schemas_resolved ?? true,
+ };
+}
+
+function makeDatabase(
+ name: string,
+ schemas: DatabaseSchema[],
+ schemas_resolved = true,
+): Database {
+ return { name, dialect: "duckdb", schemas, schemas_resolved, engine: null };
+}
+
+describe("schemaSubtreeMatchesSearch", () => {
+ it("returns false for an empty query", () => {
+ const schema = makeSchema({ name: "main", tables: [makeTable("users")] });
+ expect(schemaSubtreeMatchesSearch(schema, "")).toBe(false);
+ expect(schemaSubtreeMatchesSearch(schema, " ")).toBe(false);
+ expect(schemaSubtreeMatchesSearch(schema, undefined)).toBe(false);
+ });
+
+ it("matches a table name case-insensitively", () => {
+ const schema = makeSchema({ name: "main", tables: [makeTable("Users")] });
+ expect(schemaSubtreeMatchesSearch(schema, "user")).toBe(true);
+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(false);
+ });
+
+ it("matches a table in a resolved child schema", () => {
+ const schema = makeSchema({
+ name: "parent",
+ tables: [],
+ child_schemas: [
+ makeSchema({ name: "child", tables: [makeTable("orders")] }),
+ ],
+ });
+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(true);
+ });
+
+ it("ignores deferred tables so search never triggers a fetch", () => {
+ const schema = makeSchema({
+ name: "main",
+ tables: [makeTable("users")],
+ tables_resolved: false,
+ });
+ expect(schemaSubtreeMatchesSearch(schema, "users")).toBe(false);
+ });
+
+ it("ignores deferred child schemas", () => {
+ const schema = makeSchema({
+ name: "parent",
+ tables: [],
+ child_schemas: [
+ makeSchema({ name: "child", tables: [makeTable("orders")] }),
+ ],
+ child_schemas_resolved: false,
+ });
+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(false);
+ });
+});
+
+describe("shouldExpandDatabaseForSearch", () => {
+ it("returns false for an empty query", () => {
+ const db = makeDatabase("memory", [
+ makeSchema({ name: "main", tables: [makeTable("users")] }),
+ ]);
+ expect(shouldExpandDatabaseForSearch(db, "")).toBe(false);
+ expect(shouldExpandDatabaseForSearch(db, undefined)).toBe(false);
+ });
+
+ it("expands when a loaded schema contains a matching table", () => {
+ const db = makeDatabase("memory", [
+ makeSchema({ name: "main", tables: [makeTable("users")] }),
+ makeSchema({ name: "other", tables: [makeTable("orders")] }),
+ ]);
+ expect(shouldExpandDatabaseForSearch(db, "user")).toBe(true);
+ expect(shouldExpandDatabaseForSearch(db, "nomatch")).toBe(false);
+ });
+
+ it("does not expand when the schema list itself is deferred", () => {
+ const db = makeDatabase(
+ "memory",
+ [makeSchema({ name: "main", tables: [makeTable("users")] })],
+ /* schemas_resolved */ false,
+ );
+ expect(shouldExpandDatabaseForSearch(db, "users")).toBe(false);
+ });
+});
diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsx
index 533ce06c51..54ce7cf663 100644
--- a/frontend/src/components/datasources/datasources.tsx
+++ b/frontend/src/components/datasources/datasources.tsx
@@ -83,6 +83,8 @@ import {
areSchemasResolved,
areTablesResolved,
isSchemaless,
+ schemaSubtreeMatchesSearch,
+ shouldExpandDatabaseForSearch,
sqlCode,
tableUniqueId,
} from "./utils";
@@ -360,7 +362,6 @@ export const DataSources: React.FC = () => {
key={database.name}
connection={connection}
database={database}
- hasSearch={hasSearch}
searchValue={searchValue}
/>
))}
@@ -435,7 +436,6 @@ interface DataSourceTree {
dialect: string;
engineName: string;
databaseName: string;
- hasSearch: boolean;
searchValue?: string;
}
@@ -470,9 +470,8 @@ function buildSqlTableContext(
const DatabaseTree: React.FC<{
connection: DataSourceConnection;
database: Database;
- hasSearch: boolean;
searchValue?: string;
-}> = ({ connection, database, hasSearch, searchValue }) => {
+}> = ({ connection, database, searchValue }) => {
const tree = React.useMemo<DataSourceTree>(
() => ({
engineName: connection.name,
@@ -480,7 +479,6 @@ const DatabaseTree: React.FC<{
dialect: connection.dialect,
defaultSchema: connection.default_schema,
defaultDatabase: connection.default_database,
- hasSearch,
searchValue,
}),
[
@@ -489,7 +487,6 @@ const DatabaseTree: React.FC<{
connection.default_schema,
connection.default_database,
database.name,
- hasSearch,
searchValue,
],
);
@@ -498,7 +495,7 @@ const DatabaseTree: React.FC<{
<DatabaseItem
engineName={connection.name}
database={database}
- hasSearch={hasSearch}
+ searchValue={searchValue}
>
<DataSourceTreeContext.Provider value={tree}>
<SchemaList
@@ -513,18 +510,30 @@ const DatabaseTree: React.FC<{
};
const DatabaseItem: React.FC<{
- hasSearch: boolean;
+ searchValue?: string;
engineName: string;
database: Database;
children: React.ReactNode;
-}> = ({ hasSearch, engineName, database, children }) => {
- const [isExpanded, setIsExpanded] = React.useState(false);
+}> = ({ searchValue, engineName, database, children }) => {
+ // Auto-expand during search only when a loaded schema under this database
+ // contains a matching table.
+ const expandForSearch = shouldExpandDatabaseForSearch(database, searchValue);
+ const [isExpanded, setIsExpanded] = React.useState(expandForSearch);
const [isSelected, setIsSelected] = React.useState(false);
- const [prevHasSearch, setPrevHasSearch] = React.useState(hasSearch);
-
- if (prevHasSearch !== hasSearch) {
- setPrevHasSearch(hasSearch);
- setIsExpanded(hasSearch);
+ const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);
+ const [prevExpandForSearch, setPrevExpandForSearch] =
+ React.useState(expandForSearch);
+
+ // Re-sync when the query changes or when newly resolved data turns this
+ // branch into a match under the same query (deferred subtrees resolve
+ // asynchronously, so the decision can flip without the query changing).
+ if (
+ prevSearchValue !== searchValue ||
+ prevExpandForSearch !== expandForSearch
+ ) {
+ setPrevSearchValue(searchValue);
+ setPrevExpandForSearch(expandForSearch);
+ setIsExpanded(expandForSearch);
}
return (
@@ -657,12 +666,30 @@ interface SchemaNodeProps {
const SchemaNode: React.FC<SchemaNodeProps> = (props) => {
const { schema, schemaPath, depth } = props;
const tree = useDataSourceTree();
- const { databaseName, hasSearch, searchValue } = tree;
- const [isExpanded, setIsExpanded] = React.useState(hasSearch);
+ const { databaseName, searchValue } = tree;
+ // Auto-expand during search only when this schema's loaded subtree contains a
+ // matching table.
+ const expandForSearch = schemaSubtreeMatchesSearch(schema, searchValue);
+ const [isExpanded, setIsExpanded] = React.useState(expandForSearch);
const [isSelected, setIsSelected] = React.useState(false);
+ const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);
+ const [prevExpandForSearch, setPrevExpandForSearch] =
+ React.useState(expandForSearch);
const uniqueValue = `${databaseName}:${schemaPath.join(".")}`;
const childSchemas = schema.child_schemas ?? [];
+ // Re-sync when the query changes or when newly resolved data turns this
+ // subtree into a match under the same query (deferred tables/child schemas
+ // resolve asynchronously, so the decision can flip without the query changing).
+ if (
+ prevSearchValue !== searchValue ||
+ prevExpandForSearch !== expandForSearch
+ ) {
+ setPrevSearchValue(searchValue);
+ setPrevExpandForSearch(expandForSearch);
+ setIsExpanded(expandForSearch);
+ }
+
return (
<>
<CommandItem
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.ts
index f75753f506..904d152acb 100644
--- a/frontend/src/components/datasources/utils.ts
+++ b/frontend/src/components/datasources/utils.ts
@@ -52,6 +52,51 @@ export function areChildSchemasResolved(schema: DatabaseSchema): boolean {
return schema.child_schemas_resolved !== false;
}
+/**
+ * Whether a loaded schema subtree contains a table whose name matches
+ * `searchValue`. Deferred (not-yet-fetched) tables and child schemas are
+ * treated as non-matching, so searching never triggers a lazy catalog fetch on
+ * large/remote connections.
+ */
+export function schemaSubtreeMatchesSearch(
+ schema: DatabaseSchema,
+ searchValue: string | undefined,
+): boolean {
+ const query = searchValue?.trim().toLowerCase();
+ if (!query) {
+ return false;
+ }
+ if (
+ areTablesResolved(schema) &&
+ schema.tables.some((table) => table.name.toLowerCase().includes(query))
+ ) {
+ return true;
+ }
+ if (areChildSchemasResolved(schema)) {
+ return (schema.child_schemas ?? []).some((child) =>
+ schemaSubtreeMatchesSearch(child, query),
+ );
+ }
+ return false;
+}
+
+/**
+ * Auto-expand a database row during search only when one of its loaded schemas
+ * contains a matching table. Returns false when the schema list itself is
+ * deferred.
+ */
+export function shouldExpandDatabaseForSearch(
+ database: Database,
+ searchValue: string | undefined,
+): boolean {
+ if (!searchValue?.trim() || !areSchemasResolved(database)) {
+ return false;
+ }
+ return database.schemas.some((schema) =>
+ schemaSubtreeMatchesSearch(schema, searchValue),
+ );
+}
+
interface SqlCodeFormatter {
/**
* Format the table path based on dialect-specific rules
diff --git a/marimo/_smoke_tests/sql/catalog_children.py b/marimo/_smoke_tests/sql/catalog_children.py
new file mode 100644
index 0000000000..2c28362505
--- /dev/null
+++ b/marimo/_smoke_tests/sql/catalog_children.py
@@ -0,0 +1,685 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+# "duckdb",
+# "ibis-framework[duckdb]",
+# "marimo",
+# "pyarrow",
+# "pyiceberg[sql-sqlite]",
+# "sqlalchemy",
+# ]
+# ///
+
+# Copyright 2026 Marimo. All rights reserved.
+#
+# Smoke test for SQL catalog expansion across local/no-server engines.
+
+import marimo
+
+__generated_with = "0.23.9"
+app = marimo.App(width="medium")
+
+
+def _():
+ from datetime import date
+ from pathlib import Path
+ from textwrap import dedent
+ import sqlite3
+ import tempfile
+
+ import duckdb
+ import ibis
+ import marimo as mo
+ import pyarrow as pa
+ from pyiceberg.catalog import load_catalog
+ from sqlalchemy import create_engine
+ from sqlalchemy.pool import StaticPool
+
+ return (
+ Path,
+ StaticPool,
+ create_engine,
+ date,
+ dedent,
+ duckdb,
+ ibis,
+ load_catalog,
+ mo,
+ pa,
+ sqlite3,
+ tempfile,
+ )
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ # SQL Catalog Children Smoke Test
+
+ This notebook creates local in-memory or temporary databases with
+ nested schemas, attached catalogs/databases, views, root-level
+ tables, and PyIceberg namespaces.
+
+ ### Manual test steps
+
+ 1. Open the data sources panel after the notebook finishes running.
+ 2. Expand every connection created below: `duckdb_conn`,
+ `sqlite_conn`, `sqlalchemy_engine`, `ibis_conn`, and
+ `iceberg_catalog`.
+ 3. Verify each expansion loads immediate children once and shows
+ tables directly under engines/namespaces that expose root-level
+ tables without an empty schema wrapper.
+ """
+ )
+ )
+ return
+
+
+def _(Path, tempfile):
+ catalog_smoke_tmp = tempfile.TemporaryDirectory(prefix="marimo-catalog-smoke-")
+ catalog_smoke_root = Path(catalog_smoke_tmp.name)
+ return (catalog_smoke_root,)
+
+
[email protected](hide_code=True)
+def _(catalog_smoke_root, dedent, mo):
+ mo.md(
+ dedent(
+ f"""
+ ## Temporary Workspace
+
+ Temporary attached databases and warehouses live under:
+
+ `{catalog_smoke_root}`
+ """
+ )
+ )
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## 1. DuckDB
+
+ Creates a DuckDB in-memory connection with:
+
+ - `memory.main.root_events`
+ - `memory.retail.orders`
+ - `memory.retail.high_value_orders` view
+ - `memory.operations.inventory`
+ - attached catalog `duck_lake.bronze.events`
+ - attached catalog `duck_lake.silver.customer_rollups`
+ """
+ )
+ )
+ return
+
+
+def _(catalog_smoke_root, duckdb):
+ duckdb_conn = duckdb.connect(":memory:")
+ _duck_lake_path = catalog_smoke_root / "duck_lake.duckdb"
+ _duck_lake_sql_path = _duck_lake_path.as_posix().replace("'", "''")
+
+ duckdb_conn.execute("CREATE SCHEMA retail")
+ duckdb_conn.execute("CREATE SCHEMA operations")
+ duckdb_conn.execute(
+ """
+ CREATE TABLE root_events (
+ event_id INTEGER,
+ event_name VARCHAR,
+ payload STRUCT(region VARCHAR, amount DOUBLE)
+ )
+ """
+ )
+ duckdb_conn.execute(
+ """
+ INSERT INTO root_events VALUES
+ (1, 'opened', {'region': 'us', 'amount': 10.5}),
+ (2, 'clicked', {'region': 'eu', 'amount': 20.0})
+ """
+ )
+ duckdb_conn.execute(
+ """
+ CREATE TABLE retail.orders (
+ order_id INTEGER,
+ customer VARCHAR,
+ amount DOUBLE,
+ tags VARCHAR[]
+ )
+ """
+ )
+ duckdb_conn.execute(
+ """
+ INSERT INTO retail.orders VALUES
+ (101, 'Ada', 125.50, ['new', 'priority']),
+ (102, 'Grace', 83.00, ['returning'])
+ """
+ )
+ duckdb_conn.execute(
+ """
+ CREATE VIEW retail.high_value_orders AS
+ SELECT * FROM retail.orders WHERE amount > 100
+ """
+ )
+ duckdb_conn.execute(
+ """
+ CREATE TABLE operations.inventory (
+ sku VARCHAR,
+ warehouse VARCHAR,
+ quantity INTEGER
+ )
+ """
+ )
+ duckdb_conn.execute(
+ """
+ INSERT INTO operations.inventory VALUES
+ ('sku-1', 'west', 10),
+ ('sku-2', 'east', 25)
+ """
+ )
+ duckdb_conn.execute(f"ATTACH '{_duck_lake_sql_path}' AS duck_lake")
+ duckdb_conn.execute("CREATE SCHEMA duck_lake.bronze")
+ duckdb_conn.execute("CREATE SCHEMA duck_lake.silver")
+ duckdb_conn.execute(
+ """
+ CREATE TABLE duck_lake.bronze.events (
+ event_id INTEGER,
+ source VARCHAR,
+ received_at TIMESTAMP
+ )
+ """
+ )
+ duckdb_conn.execute(
+ """
+ INSERT INTO duck_lake.bronze.events VALUES
+ (1, 'sensor-a', CURRENT_TIMESTAMP),
+ (2, 'sensor-b', CURRENT_TIMESTAMP)
+ """
+ )
+ duckdb_conn.execute(
+ """
+ CREATE TABLE duck_lake.silver.customer_rollups (
+ customer VARCHAR,
+ total_amount DOUBLE
+ )
+ """
+ )
+ duckdb_conn.execute(
+ """
+ INSERT INTO duck_lake.silver.customer_rollups VALUES
+ ('Ada', 125.50),
+ ('Grace', 83.00)
+ """
+ )
+
+ _duckdb_tables = duckdb_conn.execute("SHOW ALL TABLES").fetchall()
+ assert any(
+ _row[0] == "memory" and _row[1] == "retail" for _row in _duckdb_tables
+ )
+ assert any(
+ _row[0] == "duck_lake" and _row[1] == "bronze" for _row in _duckdb_tables
+ )
+ return (duckdb_conn,)
+
+
+def _(duckdb_conn, mo):
+ _duckdb_preview = mo.sql(
+ f"""
+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}
+ SELECT 'duckdb' AS engine, COUNT(*) AS order_count
+ FROM retail.orders
+ """,
+ engine=duckdb_conn,
+ )
+ _duckdb_preview
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## 2. SQLite DB-API
+
+ Creates a pure `sqlite3.Connection` with:
+
+ - `main.raw_orders`
+ - attached database `tenant_a.customer_profiles`
+ - attached database `tenant_a.line_items`
+ - attached database `tenant_b.audit_log`
+ """
+ )
+ )
+ return
+
+
+def _(catalog_smoke_root, sqlite3):
+ sqlite_conn = sqlite3.connect(":memory:")
+ _tenant_a = catalog_smoke_root / "sqlite_tenant_a.db"
+ _tenant_b = catalog_smoke_root / "sqlite_tenant_b.db"
+ sqlite_conn.execute(f"ATTACH DATABASE '{_tenant_a}' AS tenant_a")
+ sqlite_conn.execute(f"ATTACH DATABASE '{_tenant_b}' AS tenant_b")
+ sqlite_conn.executescript(
+ """
+ CREATE TABLE main.raw_orders (
+ order_id INTEGER PRIMARY KEY,
+ customer TEXT,
+ amount REAL
+ );
+ INSERT INTO main.raw_orders VALUES
+ (1, 'Ada', 125.50),
+ (2, 'Grace', 83.00);
+
+ CREATE TABLE tenant_a.customer_profiles (
+ customer_id INTEGER PRIMARY KEY,
+ name TEXT,
+ preferences TEXT
+ );
+ INSERT INTO tenant_a.customer_profiles VALUES
+ (1, 'Ada', '{"tier": "gold"}'),
+ (2, 'Grace', '{"tier": "silver"}');
+
+ CREATE TABLE tenant_a.line_items (
+ order_id INTEGER,
+ sku TEXT,
+ quantity INTEGER
+ );
+ INSERT INTO tenant_a.line_items VALUES
+ (1, 'sku-1', 2),
+ (2, 'sku-2', 1);
+
+ CREATE TABLE tenant_b.audit_log (
+ audit_id INTEGER PRIMARY KEY,
+ action TEXT,
+ actor TEXT
+ );
+ INSERT INTO tenant_b.audit_log VALUES
+ (1, 'created', 'smoke-test'),
+ (2, 'updated', 'smoke-test');
+ """
+ )
+ sqlite_conn.commit()
+
+ _sqlite_schemas = [
+ _row[1] for _row in sqlite_conn.execute("PRAGMA database_list")
+ ]
+ assert {"main", "tenant_a", "tenant_b"}.issubset(_sqlite_schemas)
+ return (sqlite_conn,)
+
+
+def _(mo, raw_orders, sqlite_conn):
+ _sqlite_preview = mo.sql(
+ f"""
+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}
+ SELECT 'sqlite3' AS engine, COUNT(*) AS raw_order_count
+ FROM raw_orders
+ """,
+ engine=sqlite_conn,
+ )
+ _sqlite_preview
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## 3. SQLAlchemy SQLite
+
+ Creates a SQLAlchemy engine backed by one persistent in-memory
+ SQLite connection, with attached temporary databases:
+
+ - `main.warehouse_items`
+ - `staging.incoming_shipments`
+ - `archive.shipment_history`
+ """
+ )
+ )
+ return
+
+
+def _(StaticPool, catalog_smoke_root, create_engine):
+ sqlalchemy_engine = create_engine(
+ "sqlite:///:memory:",
+ connect_args={"check_same_thread": False},
+ poolclass=StaticPool,
+ )
+ _staging = catalog_smoke_root / "sqlalchemy_staging.db"
+ _archive = catalog_smoke_root / "sqlalchemy_archive.db"
+ with sqlalchemy_engine.begin() as _conn:
+ _conn.exec_driver_sql(f"ATTACH DATABASE '{_staging}' AS staging")
+ _conn.exec_driver_sql(f"ATTACH DATABASE '{_archive}' AS archive")
+ _conn.exec_driver_sql(
+ """
+ CREATE TABLE warehouse_items (
+ sku TEXT PRIMARY KEY,
+ name TEXT,
+ quantity INTEGER
+ )
+ """
+ )
+ _conn.exec_driver_sql(
+ """
+ INSERT INTO warehouse_items VALUES
+ ('sku-1', 'Notebook', 20),
+ ('sku-2', 'Pen', 200)
+ """
+ )
+ _conn.exec_driver_sql(
+ """
+ CREATE TABLE staging.incoming_shipments (
+ shipment_id INTEGER PRIMARY KEY,
+ sku TEXT,
+ expected_quantity INTEGER
+ )
+ """
+ )
+ _conn.exec_driver_sql(
+ """
+ INSERT INTO staging.incoming_shipments VALUES
+ (10, 'sku-1', 5),
+ (11, 'sku-3', 50)
+ """
+ )
+ _conn.exec_driver_sql(
+ """
+ CREATE TABLE archive.shipment_history (
+ shipment_id INTEGER PRIMARY KEY,
+ sku TEXT,
+ received_quantity INTEGER
+ )
+ """
+ )
+ _conn.exec_driver_sql(
+ """
+ INSERT INTO archive.shipment_history VALUES
+ (1, 'sku-1', 10),
+ (2, 'sku-2', 25)
+ """
+ )
+
+ with sqlalchemy_engine.connect() as _conn:
+ _attached = [
+ _row[1] for _row in _conn.exec_driver_sql("PRAGMA database_list")
+ ]
+ assert {"main", "staging", "archive"}.issubset(_attached)
+ return (sqlalchemy_engine,)
+
+
+def _(mo, sqlalchemy_engine, warehouse_items):
+ _sqlalchemy_preview = mo.sql(
+ f"""
+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}
+ SELECT 'sqlalchemy' AS engine, COUNT(*) AS item_count
+ FROM warehouse_items
+ """,
+ engine=sqlalchemy_engine,
+ )
+ _sqlalchemy_preview
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## 4. Ibis DuckDB Backend
+
+ Creates an Ibis DuckDB backend with:
+
+ - default `main.ibis_root_metrics`
+ - `analytics.ibis_orders`
+ - attached catalog `ibis_lake.mart.customer_features`
+ """
+ )
+ )
+ return
+
+
+def _(catalog_smoke_root, ibis, pa):
+ ibis.options.interactive = True
+ ibis_conn = ibis.duckdb.connect()
+ _ibis_lake = catalog_smoke_root / "ibis_lake.duckdb"
+ _ibis_lake_sql_path = _ibis_lake.as_posix().replace("'", "''")
+
+ ibis_conn.raw_sql("CREATE SCHEMA analytics")
+ ibis_conn.raw_sql(f"ATTACH '{_ibis_lake_sql_path}' AS ibis_lake")
+ ibis_conn.raw_sql("CREATE SCHEMA ibis_lake.mart")
+
+ _root_metrics = pa.table(
+ {
+ "metric": ["latency_ms", "throughput"],
+ "value": [42.5, 900.0],
+ }
+ )
+ _orders = pa.table(
+ {
+ "order_id": [1, 2],
+ "customer": ["Ada", "Grace"],
+ "amount": [125.5, 83.0],
+ }
+ )
+ _features = pa.table(
+ {
+ "customer": ["Ada", "Grace"],
+ "segment": ["enterprise", "startup"],
+ "score": [0.98, 0.87],
+ }
+ )
+
+ ibis_conn.create_table("ibis_root_metrics", obj=_root_metrics, overwrite=True)
+ ibis_conn.create_table(
+ "ibis_orders", obj=_orders, database="analytics", overwrite=True
+ )
+ ibis_conn.create_table(
+ "customer_features",
+ obj=_features,
+ database="ibis_lake.mart",
+ overwrite=True,
+ )
+
+ assert "ibis_root_metrics" in ibis_conn.list_tables(
+ database=("memory", "main")
+ )
+ assert "ibis_orders" in ibis_conn.list_tables(database=("memory", "analytics"))
+ assert "customer_features" in ibis_conn.list_tables(
+ database=("ibis_lake", "mart")
+ )
+ return (ibis_conn,)
+
+
+def _(ibis_conn, mo):
+ _ibis_preview = mo.sql(
+ f"""
+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}
+ SELECT 'ibis' AS engine, COUNT(*) AS order_count
+ FROM analytics.ibis_orders
+ """,
+ engine=ibis_conn,
+ )
+ _ibis_preview
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## 5. PyIceberg SQL Catalog
+
+ Creates a temporary SQLite-backed Iceberg catalog and local file
+ warehouse with nested namespaces:
+
+ - `top.root_table`
+ - `top.nested.orders`
+ - `top.nested.deep.line_items`
+ - `top.reporting.daily_rollups`
+ """
+ )
+ )
+ return
+
+
+def _(catalog_smoke_root, date, load_catalog, pa):
+ _warehouse_path = catalog_smoke_root / "iceberg_warehouse"
+ _warehouse_path.mkdir(parents=True, exist_ok=True)
+ _catalog_db = catalog_smoke_root / "pyiceberg_catalog.db"
+ iceberg_catalog = load_catalog(
+ "catalog_smoke",
+ **{
+ "type": "sql",
+ "uri": f"sqlite:///{_catalog_db}",
+ "warehouse": f"file://{_warehouse_path}",
+ },
+ )
+
+ for _namespace in [
+ "top",
+ "top.nested",
+ "top.nested.deep",
+ "top.reporting",
+ ]:
+ iceberg_catalog.create_namespace_if_not_exists(_namespace)
+
+ _root_schema = pa.schema([("id", pa.int32()), ("label", pa.string())])
+ _order_schema = pa.schema(
+ [
+ ("order_id", pa.int32()),
+ ("customer", pa.string()),
+ ("amount", pa.float64()),
+ ]
+ )
+ _line_item_schema = pa.schema(
+ [
+ ("order_id", pa.int32()),
+ ("sku", pa.string()),
+ ("quantity", pa.int32()),
+ ]
+ )
+ _rollup_schema = pa.schema(
+ [
+ ("report_date", pa.date32()),
+ ("order_count", pa.int32()),
+ ("gross_sales", pa.float64()),
+ ]
+ )
+
+ _root_table = iceberg_catalog.create_table_if_not_exists(
+ "top.root_table", schema=_root_schema
+ )
+ _orders_table = iceberg_catalog.create_table_if_not_exists(
+ "top.nested.orders", schema=_order_schema
+ )
+ _line_items_table = iceberg_catalog.create_table_if_not_exists(
+ "top.nested.deep.line_items", schema=_line_item_schema
+ )
+ _rollups_table = iceberg_catalog.create_table_if_not_exists(
+ "top.reporting.daily_rollups", schema=_rollup_schema
+ )
+
+ _root_table.overwrite(
+ pa.table(
+ {
+ "id": pa.array([1, 2], type=pa.int32()),
+ "label": ["root-a", "root-b"],
+ }
+ )
+ )
+ _orders_table.overwrite(
+ pa.table(
+ {
+ "order_id": pa.array([1, 2], type=pa.int32()),
+ "customer": ["Ada", "Grace"],
+ "amount": [125.5, 83.0],
+ }
+ )
+ )
+ _line_items_table.overwrite(
+ pa.table(
+ {
+ "order_id": pa.array([1, 1, 2], type=pa.int32()),
+ "sku": ["sku-1", "sku-2", "sku-3"],
+ "quantity": pa.array([2, 1, 4], type=pa.int32()),
+ }
+ )
+ )
+ _rollups_table.overwrite(
+ pa.table(
+ {
+ "report_date": pa.array([date(2026, 6, 13)], type=pa.date32()),
+ "order_count": pa.array([2], type=pa.int32()),
+ "gross_sales": [208.5],
+ }
+ )
+ )
+
+ assert ("top", "nested") in iceberg_catalog.list_namespaces("top")
+ assert ("top", "nested", "deep") in iceberg_catalog.list_namespaces(
+ "top.nested"
+ )
+ assert ("top", "nested", "orders") in iceberg_catalog.list_tables("top.nested")
+ return (iceberg_catalog,)
+
+
+def _(iceberg_catalog, mo):
+ _iceberg_orders = (
+ iceberg_catalog.load_table("top.nested.orders").scan().to_arrow()
+ )
+ assert _iceberg_orders.num_rows == 2
+ mo.md("**PyIceberg table load:** PASS")
+ return
+
+
[email protected](hide_code=True)
+def _(dedent, mo):
+ mo.md(
+ dedent(
+ """
+ ## Catalog Panel Checklist
+
+ Use the data sources panel to verify these expansion paths:
+
+ - `duckdb_conn`: `memory -> retail -> orders`,
+ `memory -> retail -> high_value_orders`,
+ `duck_lake -> bronze -> events`.
+ - `sqlite_conn`: `main -> raw_orders`,
+ `tenant_a -> customer_profiles`, `tenant_b -> audit_log`.
+ - `sqlalchemy_engine`: `main -> warehouse_items`,
+ `staging -> incoming_shipments`,
+ `archive -> shipment_history`.
+ - `ibis_conn`: `memory -> main -> ibis_root_metrics`,
+ `memory -> analytics -> ibis_orders`,
+ `ibis_lake -> mart -> customer_features`.
+ - `iceberg_catalog`: `top -> root_table`,
+ `top -> nested -> orders`,
+ `top -> nested -> deep -> line_items`,
+ `top -> reporting -> daily_rollups`.
+ """
+ )
+ )
+ return
+
+
+if __name__ == "__main__":
+ app.run()