A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat(datasources): only auto-expand search matches in the data sources tree (#9955)
Shahmir Varqhaea00c8519625
diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.tsindex 1d7483eb91..bad765cf5b 100644--- a/frontend/src/components/datasources/__tests__/utils.test.ts+++ b/frontend/src/components/datasources/__tests__/utils.test.ts@@ -3,8 +3,18 @@import { describe, expect, it } from "vitest";import type { SQLTableContext } from "@/core/datasets/data-source-connections";import { DUCKDB_ENGINE } from "@/core/datasets/engines";-import type { DataTable, DataTableColumn } from "@/core/kernel/messages";-import { sqlCode, tableUniqueId } from "../utils";+import type {+ Database,+ DatabaseSchema,+ DataTable,+ DataTableColumn,+} from "@/core/kernel/messages";+import {+ schemaSubtreeMatchesSearch,+ shouldExpandDatabaseForSearch,+ sqlCode,+ tableUniqueId,+} from "../utils";describe("sqlCode", () => {const mockTable: DataTable = {@@ -519,3 +529,118 @@ describe("tableUniqueId", () => {);});});++function makeTable(name: string): DataTable {+ return {+ name,+ columns: [],+ source: "memory",+ source_type: "local",+ type: "table",+ engine: null,+ indexes: null,+ num_columns: null,+ num_rows: null,+ variable_name: null,+ primary_keys: null,+ };+}++function makeSchema(opts: {+ name: string;+ tables: DataTable[];+ tables_resolved?: boolean;+ child_schemas?: DatabaseSchema[];+ child_schemas_resolved?: boolean;+}): DatabaseSchema {+ return {+ name: opts.name,+ tables: opts.tables,+ tables_resolved: opts.tables_resolved ?? true,+ child_schemas: opts.child_schemas ?? [],+ child_schemas_resolved: opts.child_schemas_resolved ?? true,+ };+}++function makeDatabase(+ name: string,+ schemas: DatabaseSchema[],+ schemas_resolved = true,+): Database {+ return { name, dialect: "duckdb", schemas, schemas_resolved, engine: null };+}++describe("schemaSubtreeMatchesSearch", () => {+ it("returns false for an empty query", () => {+ const schema = makeSchema({ name: "main", tables: [makeTable("users")] });+ expect(schemaSubtreeMatchesSearch(schema, "")).toBe(false);+ expect(schemaSubtreeMatchesSearch(schema, " ")).toBe(false);+ expect(schemaSubtreeMatchesSearch(schema, undefined)).toBe(false);+ });++ it("matches a table name case-insensitively", () => {+ const schema = makeSchema({ name: "main", tables: [makeTable("Users")] });+ expect(schemaSubtreeMatchesSearch(schema, "user")).toBe(true);+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(false);+ });++ it("matches a table in a resolved child schema", () => {+ const schema = makeSchema({+ name: "parent",+ tables: [],+ child_schemas: [+ makeSchema({ name: "child", tables: [makeTable("orders")] }),+ ],+ });+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(true);+ });++ it("ignores deferred tables so search never triggers a fetch", () => {+ const schema = makeSchema({+ name: "main",+ tables: [makeTable("users")],+ tables_resolved: false,+ });+ expect(schemaSubtreeMatchesSearch(schema, "users")).toBe(false);+ });++ it("ignores deferred child schemas", () => {+ const schema = makeSchema({+ name: "parent",+ tables: [],+ child_schemas: [+ makeSchema({ name: "child", tables: [makeTable("orders")] }),+ ],+ child_schemas_resolved: false,+ });+ expect(schemaSubtreeMatchesSearch(schema, "orders")).toBe(false);+ });+});++describe("shouldExpandDatabaseForSearch", () => {+ it("returns false for an empty query", () => {+ const db = makeDatabase("memory", [+ makeSchema({ name: "main", tables: [makeTable("users")] }),+ ]);+ expect(shouldExpandDatabaseForSearch(db, "")).toBe(false);+ expect(shouldExpandDatabaseForSearch(db, undefined)).toBe(false);+ });++ it("expands when a loaded schema contains a matching table", () => {+ const db = makeDatabase("memory", [+ makeSchema({ name: "main", tables: [makeTable("users")] }),+ makeSchema({ name: "other", tables: [makeTable("orders")] }),+ ]);+ expect(shouldExpandDatabaseForSearch(db, "user")).toBe(true);+ expect(shouldExpandDatabaseForSearch(db, "nomatch")).toBe(false);+ });++ it("does not expand when the schema list itself is deferred", () => {+ const db = makeDatabase(+ "memory",+ [makeSchema({ name: "main", tables: [makeTable("users")] })],+ /* schemas_resolved */ false,+ );+ expect(shouldExpandDatabaseForSearch(db, "users")).toBe(false);+ });+});diff --git a/frontend/src/components/datasources/datasources.tsx b/frontend/src/components/datasources/datasources.tsxindex 533ce06c51..54ce7cf663 100644--- a/frontend/src/components/datasources/datasources.tsx+++ b/frontend/src/components/datasources/datasources.tsx@@ -83,6 +83,8 @@ import {areSchemasResolved,areTablesResolved,isSchemaless,+ schemaSubtreeMatchesSearch,+ shouldExpandDatabaseForSearch,sqlCode,tableUniqueId,} from "./utils";@@ -360,7 +362,6 @@ export const DataSources: React.FC = () => {key={database.name}connection={connection}database={database}- hasSearch={hasSearch}searchValue={searchValue}/>))}@@ -435,7 +436,6 @@ interface DataSourceTree {dialect: string;engineName: string;databaseName: string;- hasSearch: boolean;searchValue?: string;}@@ -470,9 +470,8 @@ function buildSqlTableContext(const DatabaseTree: React.FC<{connection: DataSourceConnection;database: Database;- hasSearch: boolean;searchValue?: string;-}> = ({ connection, database, hasSearch, searchValue }) => {+}> = ({ connection, database, searchValue }) => {const tree = React.useMemo<DataSourceTree>(() => ({engineName: connection.name,@@ -480,7 +479,6 @@ const DatabaseTree: React.FC<{dialect: connection.dialect,defaultSchema: connection.default_schema,defaultDatabase: connection.default_database,- hasSearch,searchValue,}),[@@ -489,7 +487,6 @@ const DatabaseTree: React.FC<{connection.default_schema,connection.default_database,database.name,- hasSearch,searchValue,],);@@ -498,7 +495,7 @@ const DatabaseTree: React.FC<{<DatabaseItemengineName={connection.name}database={database}- hasSearch={hasSearch}+ searchValue={searchValue}><DataSourceTreeContext.Provider value={tree}><SchemaList@@ -513,18 +510,30 @@ const DatabaseTree: React.FC<{};const DatabaseItem: React.FC<{- hasSearch: boolean;+ searchValue?: string;engineName: string;database: Database;children: React.ReactNode;-}> = ({ hasSearch, engineName, database, children }) => {- const [isExpanded, setIsExpanded] = React.useState(false);+}> = ({ searchValue, engineName, database, children }) => {+ // Auto-expand during search only when a loaded schema under this database+ // contains a matching table.+ const expandForSearch = shouldExpandDatabaseForSearch(database, searchValue);+ const [isExpanded, setIsExpanded] = React.useState(expandForSearch);const [isSelected, setIsSelected] = React.useState(false);- const [prevHasSearch, setPrevHasSearch] = React.useState(hasSearch);-- if (prevHasSearch !== hasSearch) {- setPrevHasSearch(hasSearch);- setIsExpanded(hasSearch);+ const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);+ const [prevExpandForSearch, setPrevExpandForSearch] =+ React.useState(expandForSearch);++ // Re-sync when the query changes or when newly resolved data turns this+ // branch into a match under the same query (deferred subtrees resolve+ // asynchronously, so the decision can flip without the query changing).+ if (+ prevSearchValue !== searchValue ||+ prevExpandForSearch !== expandForSearch+ ) {+ setPrevSearchValue(searchValue);+ setPrevExpandForSearch(expandForSearch);+ setIsExpanded(expandForSearch);}return (@@ -657,12 +666,30 @@ interface SchemaNodeProps {const SchemaNode: React.FC<SchemaNodeProps> = (props) => {const { schema, schemaPath, depth } = props;const tree = useDataSourceTree();- const { databaseName, hasSearch, searchValue } = tree;- const [isExpanded, setIsExpanded] = React.useState(hasSearch);+ const { databaseName, searchValue } = tree;+ // Auto-expand during search only when this schema's loaded subtree contains a+ // matching table.+ const expandForSearch = schemaSubtreeMatchesSearch(schema, searchValue);+ const [isExpanded, setIsExpanded] = React.useState(expandForSearch);const [isSelected, setIsSelected] = React.useState(false);+ const [prevSearchValue, setPrevSearchValue] = React.useState(searchValue);+ const [prevExpandForSearch, setPrevExpandForSearch] =+ React.useState(expandForSearch);const uniqueValue = `${databaseName}:${schemaPath.join(".")}`;const childSchemas = schema.child_schemas ?? [];+ // Re-sync when the query changes or when newly resolved data turns this+ // subtree into a match under the same query (deferred tables/child schemas+ // resolve asynchronously, so the decision can flip without the query changing).+ if (+ prevSearchValue !== searchValue ||+ prevExpandForSearch !== expandForSearch+ ) {+ setPrevSearchValue(searchValue);+ setPrevExpandForSearch(expandForSearch);+ setIsExpanded(expandForSearch);+ }+return (<><CommandItemdiff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.tsindex f75753f506..904d152acb 100644--- a/frontend/src/components/datasources/utils.ts+++ b/frontend/src/components/datasources/utils.ts@@ -52,6 +52,51 @@ export function areChildSchemasResolved(schema: DatabaseSchema): boolean {return schema.child_schemas_resolved !== false;}+/**+ * Whether a loaded schema subtree contains a table whose name matches+ * `searchValue`. Deferred (not-yet-fetched) tables and child schemas are+ * treated as non-matching, so searching never triggers a lazy catalog fetch on+ * large/remote connections.+ */+export function schemaSubtreeMatchesSearch(+ schema: DatabaseSchema,+ searchValue: string | undefined,+): boolean {+ const query = searchValue?.trim().toLowerCase();+ if (!query) {+ return false;+ }+ if (+ areTablesResolved(schema) &&+ schema.tables.some((table) => table.name.toLowerCase().includes(query))+ ) {+ return true;+ }+ if (areChildSchemasResolved(schema)) {+ return (schema.child_schemas ?? []).some((child) =>+ schemaSubtreeMatchesSearch(child, query),+ );+ }+ return false;+}++/**+ * Auto-expand a database row during search only when one of its loaded schemas+ * contains a matching table. Returns false when the schema list itself is+ * deferred.+ */+export function shouldExpandDatabaseForSearch(+ database: Database,+ searchValue: string | undefined,+): boolean {+ if (!searchValue?.trim() || !areSchemasResolved(database)) {+ return false;+ }+ return database.schemas.some((schema) =>+ schemaSubtreeMatchesSearch(schema, searchValue),+ );+}+interface SqlCodeFormatter {/*** Format the table path based on dialect-specific rulesdiff --git a/marimo/_smoke_tests/sql/catalog_children.py b/marimo/_smoke_tests/sql/catalog_children.pynew file mode 100644index 0000000000..2c28362505--- /dev/null+++ b/marimo/_smoke_tests/sql/catalog_children.py@@ -0,0 +1,685 @@+# /// script+# requires-python = ">=3.12"+# dependencies = [+# "duckdb",+# "ibis-framework[duckdb]",+# "marimo",+# "pyarrow",+# "pyiceberg[sql-sqlite]",+# "sqlalchemy",+# ]+# ///++# Copyright 2026 Marimo. All rights reserved.+#+# Smoke test for SQL catalog expansion across local/no-server engines.++import marimo++__generated_with = "0.23.9"+app = marimo.App(width="medium")+++def _():+ from datetime import date+ from pathlib import Path+ from textwrap import dedent+ import sqlite3+ import tempfile++ import duckdb+ import ibis+ import marimo as mo+ import pyarrow as pa+ from pyiceberg.catalog import load_catalog+ from sqlalchemy import create_engine+ from sqlalchemy.pool import StaticPool++ return (+ Path,+ StaticPool,+ create_engine,+ date,+ dedent,+ duckdb,+ ibis,+ load_catalog,+ mo,+ pa,+ sqlite3,+ tempfile,+ )++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ # SQL Catalog Children Smoke Test++ This notebook creates local in-memory or temporary databases with+ nested schemas, attached catalogs/databases, views, root-level+ tables, and PyIceberg namespaces.++ ### Manual test steps++ 1. Open the data sources panel after the notebook finishes running.+ 2. Expand every connection created below: `duckdb_conn`,+ `sqlite_conn`, `sqlalchemy_engine`, `ibis_conn`, and+ `iceberg_catalog`.+ 3. Verify each expansion loads immediate children once and shows+ tables directly under engines/namespaces that expose root-level+ tables without an empty schema wrapper.+ """+ )+ )+ return+++def _(Path, tempfile):+ catalog_smoke_tmp = tempfile.TemporaryDirectory(prefix="marimo-catalog-smoke-")+ catalog_smoke_root = Path(catalog_smoke_tmp.name)+ return (catalog_smoke_root,)++[email protected](hide_code=True)+def _(catalog_smoke_root, dedent, mo):+ mo.md(+ dedent(+ f"""+ ## Temporary Workspace++ Temporary attached databases and warehouses live under:++ `{catalog_smoke_root}`+ """+ )+ )+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## 1. DuckDB++ Creates a DuckDB in-memory connection with:++ - `memory.main.root_events`+ - `memory.retail.orders`+ - `memory.retail.high_value_orders` view+ - `memory.operations.inventory`+ - attached catalog `duck_lake.bronze.events`+ - attached catalog `duck_lake.silver.customer_rollups`+ """+ )+ )+ return+++def _(catalog_smoke_root, duckdb):+ duckdb_conn = duckdb.connect(":memory:")+ _duck_lake_path = catalog_smoke_root / "duck_lake.duckdb"+ _duck_lake_sql_path = _duck_lake_path.as_posix().replace("'", "''")++ duckdb_conn.execute("CREATE SCHEMA retail")+ duckdb_conn.execute("CREATE SCHEMA operations")+ duckdb_conn.execute(+ """+ CREATE TABLE root_events (+ event_id INTEGER,+ event_name VARCHAR,+ payload STRUCT(region VARCHAR, amount DOUBLE)+ )+ """+ )+ duckdb_conn.execute(+ """+ INSERT INTO root_events VALUES+ (1, 'opened', {'region': 'us', 'amount': 10.5}),+ (2, 'clicked', {'region': 'eu', 'amount': 20.0})+ """+ )+ duckdb_conn.execute(+ """+ CREATE TABLE retail.orders (+ order_id INTEGER,+ customer VARCHAR,+ amount DOUBLE,+ tags VARCHAR[]+ )+ """+ )+ duckdb_conn.execute(+ """+ INSERT INTO retail.orders VALUES+ (101, 'Ada', 125.50, ['new', 'priority']),+ (102, 'Grace', 83.00, ['returning'])+ """+ )+ duckdb_conn.execute(+ """+ CREATE VIEW retail.high_value_orders AS+ SELECT * FROM retail.orders WHERE amount > 100+ """+ )+ duckdb_conn.execute(+ """+ CREATE TABLE operations.inventory (+ sku VARCHAR,+ warehouse VARCHAR,+ quantity INTEGER+ )+ """+ )+ duckdb_conn.execute(+ """+ INSERT INTO operations.inventory VALUES+ ('sku-1', 'west', 10),+ ('sku-2', 'east', 25)+ """+ )+ duckdb_conn.execute(f"ATTACH '{_duck_lake_sql_path}' AS duck_lake")+ duckdb_conn.execute("CREATE SCHEMA duck_lake.bronze")+ duckdb_conn.execute("CREATE SCHEMA duck_lake.silver")+ duckdb_conn.execute(+ """+ CREATE TABLE duck_lake.bronze.events (+ event_id INTEGER,+ source VARCHAR,+ received_at TIMESTAMP+ )+ """+ )+ duckdb_conn.execute(+ """+ INSERT INTO duck_lake.bronze.events VALUES+ (1, 'sensor-a', CURRENT_TIMESTAMP),+ (2, 'sensor-b', CURRENT_TIMESTAMP)+ """+ )+ duckdb_conn.execute(+ """+ CREATE TABLE duck_lake.silver.customer_rollups (+ customer VARCHAR,+ total_amount DOUBLE+ )+ """+ )+ duckdb_conn.execute(+ """+ INSERT INTO duck_lake.silver.customer_rollups VALUES+ ('Ada', 125.50),+ ('Grace', 83.00)+ """+ )++ _duckdb_tables = duckdb_conn.execute("SHOW ALL TABLES").fetchall()+ assert any(+ _row[0] == "memory" and _row[1] == "retail" for _row in _duckdb_tables+ )+ assert any(+ _row[0] == "duck_lake" and _row[1] == "bronze" for _row in _duckdb_tables+ )+ return (duckdb_conn,)+++def _(duckdb_conn, mo):+ _duckdb_preview = mo.sql(+ f"""+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}+ SELECT 'duckdb' AS engine, COUNT(*) AS order_count+ FROM retail.orders+ """,+ engine=duckdb_conn,+ )+ _duckdb_preview+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## 2. SQLite DB-API++ Creates a pure `sqlite3.Connection` with:++ - `main.raw_orders`+ - attached database `tenant_a.customer_profiles`+ - attached database `tenant_a.line_items`+ - attached database `tenant_b.audit_log`+ """+ )+ )+ return+++def _(catalog_smoke_root, sqlite3):+ sqlite_conn = sqlite3.connect(":memory:")+ _tenant_a = catalog_smoke_root / "sqlite_tenant_a.db"+ _tenant_b = catalog_smoke_root / "sqlite_tenant_b.db"+ sqlite_conn.execute(f"ATTACH DATABASE '{_tenant_a}' AS tenant_a")+ sqlite_conn.execute(f"ATTACH DATABASE '{_tenant_b}' AS tenant_b")+ sqlite_conn.executescript(+ """+ CREATE TABLE main.raw_orders (+ order_id INTEGER PRIMARY KEY,+ customer TEXT,+ amount REAL+ );+ INSERT INTO main.raw_orders VALUES+ (1, 'Ada', 125.50),+ (2, 'Grace', 83.00);++ CREATE TABLE tenant_a.customer_profiles (+ customer_id INTEGER PRIMARY KEY,+ name TEXT,+ preferences TEXT+ );+ INSERT INTO tenant_a.customer_profiles VALUES+ (1, 'Ada', '{"tier": "gold"}'),+ (2, 'Grace', '{"tier": "silver"}');++ CREATE TABLE tenant_a.line_items (+ order_id INTEGER,+ sku TEXT,+ quantity INTEGER+ );+ INSERT INTO tenant_a.line_items VALUES+ (1, 'sku-1', 2),+ (2, 'sku-2', 1);++ CREATE TABLE tenant_b.audit_log (+ audit_id INTEGER PRIMARY KEY,+ action TEXT,+ actor TEXT+ );+ INSERT INTO tenant_b.audit_log VALUES+ (1, 'created', 'smoke-test'),+ (2, 'updated', 'smoke-test');+ """+ )+ sqlite_conn.commit()++ _sqlite_schemas = [+ _row[1] for _row in sqlite_conn.execute("PRAGMA database_list")+ ]+ assert {"main", "tenant_a", "tenant_b"}.issubset(_sqlite_schemas)+ return (sqlite_conn,)+++def _(mo, raw_orders, sqlite_conn):+ _sqlite_preview = mo.sql(+ f"""+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}+ SELECT 'sqlite3' AS engine, COUNT(*) AS raw_order_count+ FROM raw_orders+ """,+ engine=sqlite_conn,+ )+ _sqlite_preview+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## 3. SQLAlchemy SQLite++ Creates a SQLAlchemy engine backed by one persistent in-memory+ SQLite connection, with attached temporary databases:++ - `main.warehouse_items`+ - `staging.incoming_shipments`+ - `archive.shipment_history`+ """+ )+ )+ return+++def _(StaticPool, catalog_smoke_root, create_engine):+ sqlalchemy_engine = create_engine(+ "sqlite:///:memory:",+ connect_args={"check_same_thread": False},+ poolclass=StaticPool,+ )+ _staging = catalog_smoke_root / "sqlalchemy_staging.db"+ _archive = catalog_smoke_root / "sqlalchemy_archive.db"+ with sqlalchemy_engine.begin() as _conn:+ _conn.exec_driver_sql(f"ATTACH DATABASE '{_staging}' AS staging")+ _conn.exec_driver_sql(f"ATTACH DATABASE '{_archive}' AS archive")+ _conn.exec_driver_sql(+ """+ CREATE TABLE warehouse_items (+ sku TEXT PRIMARY KEY,+ name TEXT,+ quantity INTEGER+ )+ """+ )+ _conn.exec_driver_sql(+ """+ INSERT INTO warehouse_items VALUES+ ('sku-1', 'Notebook', 20),+ ('sku-2', 'Pen', 200)+ """+ )+ _conn.exec_driver_sql(+ """+ CREATE TABLE staging.incoming_shipments (+ shipment_id INTEGER PRIMARY KEY,+ sku TEXT,+ expected_quantity INTEGER+ )+ """+ )+ _conn.exec_driver_sql(+ """+ INSERT INTO staging.incoming_shipments VALUES+ (10, 'sku-1', 5),+ (11, 'sku-3', 50)+ """+ )+ _conn.exec_driver_sql(+ """+ CREATE TABLE archive.shipment_history (+ shipment_id INTEGER PRIMARY KEY,+ sku TEXT,+ received_quantity INTEGER+ )+ """+ )+ _conn.exec_driver_sql(+ """+ INSERT INTO archive.shipment_history VALUES+ (1, 'sku-1', 10),+ (2, 'sku-2', 25)+ """+ )++ with sqlalchemy_engine.connect() as _conn:+ _attached = [+ _row[1] for _row in _conn.exec_driver_sql("PRAGMA database_list")+ ]+ assert {"main", "staging", "archive"}.issubset(_attached)+ return (sqlalchemy_engine,)+++def _(mo, sqlalchemy_engine, warehouse_items):+ _sqlalchemy_preview = mo.sql(+ f"""+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}+ SELECT 'sqlalchemy' AS engine, COUNT(*) AS item_count+ FROM warehouse_items+ """,+ engine=sqlalchemy_engine,+ )+ _sqlalchemy_preview+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## 4. Ibis DuckDB Backend++ Creates an Ibis DuckDB backend with:++ - default `main.ibis_root_metrics`+ - `analytics.ibis_orders`+ - attached catalog `ibis_lake.mart.customer_features`+ """+ )+ )+ return+++def _(catalog_smoke_root, ibis, pa):+ ibis.options.interactive = True+ ibis_conn = ibis.duckdb.connect()+ _ibis_lake = catalog_smoke_root / "ibis_lake.duckdb"+ _ibis_lake_sql_path = _ibis_lake.as_posix().replace("'", "''")++ ibis_conn.raw_sql("CREATE SCHEMA analytics")+ ibis_conn.raw_sql(f"ATTACH '{_ibis_lake_sql_path}' AS ibis_lake")+ ibis_conn.raw_sql("CREATE SCHEMA ibis_lake.mart")++ _root_metrics = pa.table(+ {+ "metric": ["latency_ms", "throughput"],+ "value": [42.5, 900.0],+ }+ )+ _orders = pa.table(+ {+ "order_id": [1, 2],+ "customer": ["Ada", "Grace"],+ "amount": [125.5, 83.0],+ }+ )+ _features = pa.table(+ {+ "customer": ["Ada", "Grace"],+ "segment": ["enterprise", "startup"],+ "score": [0.98, 0.87],+ }+ )++ ibis_conn.create_table("ibis_root_metrics", obj=_root_metrics, overwrite=True)+ ibis_conn.create_table(+ "ibis_orders", obj=_orders, database="analytics", overwrite=True+ )+ ibis_conn.create_table(+ "customer_features",+ obj=_features,+ database="ibis_lake.mart",+ overwrite=True,+ )++ assert "ibis_root_metrics" in ibis_conn.list_tables(+ database=("memory", "main")+ )+ assert "ibis_orders" in ibis_conn.list_tables(database=("memory", "analytics"))+ assert "customer_features" in ibis_conn.list_tables(+ database=("ibis_lake", "mart")+ )+ return (ibis_conn,)+++def _(ibis_conn, mo):+ _ibis_preview = mo.sql(+ f"""+ -- Keep this as an f-string so SQL identifiers are not notebook deps: {""}+ SELECT 'ibis' AS engine, COUNT(*) AS order_count+ FROM analytics.ibis_orders+ """,+ engine=ibis_conn,+ )+ _ibis_preview+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## 5. PyIceberg SQL Catalog++ Creates a temporary SQLite-backed Iceberg catalog and local file+ warehouse with nested namespaces:++ - `top.root_table`+ - `top.nested.orders`+ - `top.nested.deep.line_items`+ - `top.reporting.daily_rollups`+ """+ )+ )+ return+++def _(catalog_smoke_root, date, load_catalog, pa):+ _warehouse_path = catalog_smoke_root / "iceberg_warehouse"+ _warehouse_path.mkdir(parents=True, exist_ok=True)+ _catalog_db = catalog_smoke_root / "pyiceberg_catalog.db"+ iceberg_catalog = load_catalog(+ "catalog_smoke",+ **{+ "type": "sql",+ "uri": f"sqlite:///{_catalog_db}",+ "warehouse": f"file://{_warehouse_path}",+ },+ )++ for _namespace in [+ "top",+ "top.nested",+ "top.nested.deep",+ "top.reporting",+ ]:+ iceberg_catalog.create_namespace_if_not_exists(_namespace)++ _root_schema = pa.schema([("id", pa.int32()), ("label", pa.string())])+ _order_schema = pa.schema(+ [+ ("order_id", pa.int32()),+ ("customer", pa.string()),+ ("amount", pa.float64()),+ ]+ )+ _line_item_schema = pa.schema(+ [+ ("order_id", pa.int32()),+ ("sku", pa.string()),+ ("quantity", pa.int32()),+ ]+ )+ _rollup_schema = pa.schema(+ [+ ("report_date", pa.date32()),+ ("order_count", pa.int32()),+ ("gross_sales", pa.float64()),+ ]+ )++ _root_table = iceberg_catalog.create_table_if_not_exists(+ "top.root_table", schema=_root_schema+ )+ _orders_table = iceberg_catalog.create_table_if_not_exists(+ "top.nested.orders", schema=_order_schema+ )+ _line_items_table = iceberg_catalog.create_table_if_not_exists(+ "top.nested.deep.line_items", schema=_line_item_schema+ )+ _rollups_table = iceberg_catalog.create_table_if_not_exists(+ "top.reporting.daily_rollups", schema=_rollup_schema+ )++ _root_table.overwrite(+ pa.table(+ {+ "id": pa.array([1, 2], type=pa.int32()),+ "label": ["root-a", "root-b"],+ }+ )+ )+ _orders_table.overwrite(+ pa.table(+ {+ "order_id": pa.array([1, 2], type=pa.int32()),+ "customer": ["Ada", "Grace"],+ "amount": [125.5, 83.0],+ }+ )+ )+ _line_items_table.overwrite(+ pa.table(+ {+ "order_id": pa.array([1, 1, 2], type=pa.int32()),+ "sku": ["sku-1", "sku-2", "sku-3"],+ "quantity": pa.array([2, 1, 4], type=pa.int32()),+ }+ )+ )+ _rollups_table.overwrite(+ pa.table(+ {+ "report_date": pa.array([date(2026, 6, 13)], type=pa.date32()),+ "order_count": pa.array([2], type=pa.int32()),+ "gross_sales": [208.5],+ }+ )+ )++ assert ("top", "nested") in iceberg_catalog.list_namespaces("top")+ assert ("top", "nested", "deep") in iceberg_catalog.list_namespaces(+ "top.nested"+ )+ assert ("top", "nested", "orders") in iceberg_catalog.list_tables("top.nested")+ return (iceberg_catalog,)+++def _(iceberg_catalog, mo):+ _iceberg_orders = (+ iceberg_catalog.load_table("top.nested.orders").scan().to_arrow()+ )+ assert _iceberg_orders.num_rows == 2+ mo.md("**PyIceberg table load:** PASS")+ return++[email protected](hide_code=True)+def _(dedent, mo):+ mo.md(+ dedent(+ """+ ## Catalog Panel Checklist++ Use the data sources panel to verify these expansion paths:++ - `duckdb_conn`: `memory -> retail -> orders`,+ `memory -> retail -> high_value_orders`,+ `duck_lake -> bronze -> events`.+ - `sqlite_conn`: `main -> raw_orders`,+ `tenant_a -> customer_profiles`, `tenant_b -> audit_log`.+ - `sqlalchemy_engine`: `main -> warehouse_items`,+ `staging -> incoming_shipments`,+ `archive -> shipment_history`.+ - `ibis_conn`: `memory -> main -> ibis_root_metrics`,+ `memory -> analytics -> ibis_orders`,+ `ibis_lake -> mart -> customer_features`.+ - `iceberg_catalog`: `top -> root_table`,+ `top -> nested -> orders`,+ `top -> nested -> deep -> line_items`,+ `top -> reporting -> daily_rollups`.+ """+ )+ )+ return+++if __name__ == "__main__":+ app.run()