Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

update datasource connection when manually introspecting (#6968)

Shahmir Varqha09fdbf2b0c1a
diff --git a/marimo/_messaging/ops.py b/marimo/_messaging/ops.py
index e5d3fe8318..f27ab16d07 100644
--- a/marimo/_messaging/ops.py
+++ b/marimo/_messaging/ops.py
@@ -587,11 +587,20 @@ class Datasets(Op, tag="datasets"):
clear_channel: Optional[DataTableSource] = None
+class SQLMetadata(msgspec.Struct, tag="sql-metadata"):
+ """Metadata for a SQL database."""
+
+ connection: str
+ database: str
+ schema: str
+
+
class SQLTablePreview(Op, tag="sql-table-preview"):
"""Preview of a table in a SQL database."""
name: ClassVar[str] = "sql-table-preview"
request_id: RequestId
+ metadata: SQLMetadata
table: Optional[DataTable]
error: Optional[str] = None
@@ -601,6 +610,7 @@ class SQLTableListPreview(Op, tag="sql-table-list-preview"):
name: ClassVar[str] = "sql-table-list-preview"
request_id: RequestId
+ metadata: SQLMetadata
tables: list[DataTable] = msgspec.field(default_factory=list)
error: Optional[str] = None
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index 96c75c2b46..09fd1f3cc4 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -67,6 +67,7 @@ from marimo._messaging.ops import (
PackageStatusType,
RemoveUIElements,
SecretKeysResult,
+ SQLMetadata,
SQLTableListPreview,
SQLTablePreview,
ValidateSQLResult,
@@ -2454,11 +2455,19 @@ class DatasetCallbacks:
database_name = request.database
schema_name = request.schema
table_name = request.table_name
+ sql_metadata = SQLMetadata(
+ connection=variable_name,
+ database=database_name,
+ schema=schema_name,
+ )
engine, error = self.get_engine_catalog(variable_name)
if error is not None or engine is None:
SQLTablePreview(
- request_id=request.request_id, table=None, error=error
+ request_id=request.request_id,
+ table=None,
+ error=error,
+ metadata=sql_metadata,
).broadcast()
return
@@ -2470,7 +2479,9 @@ class DatasetCallbacks:
)
SQLTablePreview(
- request_id=request.request_id, table=table
+ request_id=request.request_id,
+ table=table,
+ metadata=sql_metadata,
).broadcast()
except Exception as e:
LOGGER.exception(
@@ -2482,6 +2493,7 @@ class DatasetCallbacks:
request_id=request.request_id,
table=None,
error="Failed to get table details: " + str(e),
+ metadata=sql_metadata,
).broadcast()
@kernel_tracer.start_as_current_span("preview_sql_table_list")
@@ -2499,11 +2511,19 @@ class DatasetCallbacks:
variable_name = cast(VariableName, request.engine)
database_name = request.database
schema_name = request.schema
+ sql_metadata = SQLMetadata(
+ connection=variable_name,
+ database=database_name,
+ schema=schema_name,
+ )
engine, error = self.get_engine_catalog(variable_name)
if error is not None or engine is None:
SQLTableListPreview(
- request_id=request.request_id, tables=[], error=error
+ request_id=request.request_id,
+ tables=[],
+ error=error,
+ metadata=sql_metadata,
).broadcast()
return
@@ -2514,7 +2534,9 @@ class DatasetCallbacks:
include_table_details=False,
)
SQLTableListPreview(
- request_id=request.request_id, tables=table_list
+ request_id=request.request_id,
+ tables=table_list,
+ metadata=sql_metadata,
).broadcast()
except Exception as e:
LOGGER.exception(
@@ -2524,7 +2546,8 @@ class DatasetCallbacks:
request_id=request.request_id,
tables=[],
error="Failed to get table list: " + str(e),
- )
+ metadata=sql_metadata,
+ ).broadcast()
@kernel_tracer.start_as_current_span("preview_datasource_connection")
async def preview_datasource_connection(
diff --git a/marimo/_server/session/session_view.py b/marimo/_server/session/session_view.py
index f21957b52e..4b8deb57b5 100644
--- a/marimo/_server/session/session_view.py
+++ b/marimo/_server/session/session_view.py
@@ -15,6 +15,8 @@ from marimo._messaging.ops import (
Interrupted,
MessageOperation,
SendUIElementMessage,
+ SQLTableListPreview,
+ SQLTablePreview,
StartupLogs,
UpdateCellCodes,
UpdateCellIdsRequest,
@@ -32,6 +34,10 @@ from marimo._runtime.requests import (
SetUIElementValueRequest,
SyncGraphRequest,
)
+from marimo._sql.connection_utils import (
+ update_table_in_connection,
+ update_table_list_in_connection,
+)
from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE
from marimo._types.ids import CellId_t, WidgetModelId
from marimo._utils.lists import as_list
@@ -248,6 +254,27 @@ class SessionView:
connections=list(connections.values())
)
+ elif isinstance(operation, SQLTablePreview):
+ sql_table_preview = operation
+ sql_metadata = sql_table_preview.metadata
+ table_preview_connections = self.data_connectors.connections
+ if sql_table_preview.table is not None:
+ update_table_in_connection(
+ table_preview_connections,
+ sql_metadata,
+ sql_table_preview.table,
+ )
+
+ elif isinstance(operation, SQLTableListPreview):
+ sql_table_list_preview = operation
+ sql_metadata = sql_table_list_preview.metadata
+ table_list_connections = self.data_connectors.connections
+ update_table_list_in_connection(
+ table_list_connections,
+ sql_metadata,
+ sql_table_list_preview.tables,
+ )
+
elif isinstance(operation, UpdateCellIdsRequest):
self.cell_ids = operation
diff --git a/marimo/_sql/connection_utils.py b/marimo/_sql/connection_utils.py
new file mode 100644
index 0000000000..52b1be8a7d
--- /dev/null
+++ b/marimo/_sql/connection_utils.py
@@ -0,0 +1,62 @@
+# Copyright 2025 Marimo. All rights reserved.
+
+from marimo._data.models import DataSourceConnection, DataTable
+from marimo._messaging.ops import SQLMetadata
+
+
+def update_table_in_connection(
+ connections: list[DataSourceConnection],
+ sql_metadata: SQLMetadata,
+ updated_table: DataTable,
+) -> None:
+ """Update a table in the connection hierarchy in-place
+
+ Args:
+ connections: List of data source connections
+ sql_metadata: SQL metadata containing connection, database, schema info
+ updated_table: The updated table to replace the existing one
+ """
+ for connection in connections:
+ if connection.name != sql_metadata.connection:
+ continue
+
+ for database in connection.databases:
+ if database.name != sql_metadata.database:
+ continue
+
+ for schema in database.schemas:
+ if schema.name != sql_metadata.schema:
+ continue
+
+ for i, table in enumerate(schema.tables):
+ if table.name == updated_table.name:
+ schema.tables[i] = updated_table
+ return
+
+
+def update_table_list_in_connection(
+ connections: list[DataSourceConnection],
+ sql_metadata: SQLMetadata,
+ updated_table_list: list[DataTable],
+) -> None:
+ """Update a list of tables in the connection hierarchy, updates in-place.
+
+ Args:
+ connections: List of data source connections
+ sql_metadata: SQL metadata containing connection, database, schema info
+ updated_table_list: The updated list of tables to replace the existing ones
+ """
+ for connection in connections:
+ if connection.name != sql_metadata.connection:
+ continue
+
+ for database in connection.databases:
+ if database.name != sql_metadata.database:
+ continue
+
+ for schema in database.schemas:
+ if schema.name != sql_metadata.schema:
+ continue
+
+ schema.tables = updated_table_list
+ return
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index ca50d00d1d..545cee42b6 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -2897,6 +2897,25 @@ components:
- watcher_on_save
title: RuntimeConfig
type: object
+ SQLMetadata:
+ description: Metadata for a SQL database.
+ properties:
+ connection:
+ type: string
+ database:
+ type: string
+ schema:
+ type: string
+ type:
+ enum:
+ - sql-metadata
+ required:
+ - type
+ - connection
+ - database
+ - schema
+ title: SQLMetadata
+ type: object
SQLTableListPreview:
description: Preview of a list of tables in a schema.
properties:
@@ -2905,6 +2924,8 @@ components:
- type: string
- type: 'null'
default: null
+ metadata:
+ $ref: '#/components/schemas/SQLMetadata'
op:
enum:
- sql-table-list-preview
@@ -2918,6 +2939,7 @@ components:
required:
- op
- request_id
+ - metadata
title: SQLTableListPreview
type: object
SQLTablePreview:
@@ -2928,6 +2950,8 @@ components:
- type: string
- type: 'null'
default: null
+ metadata:
+ $ref: '#/components/schemas/SQLMetadata'
op:
enum:
- sql-table-preview
@@ -2940,6 +2964,7 @@ components:
required:
- op
- request_id
+ - metadata
- table
title: SQLTablePreview
type: object
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 74f70a0f97..a6889753ae 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4346,6 +4346,17 @@ export interface components {
/** @enum {unknown} */
watcher_on_save: "autorun" | "lazy";
};
+ /**
+ * SQLMetadata
+ * @description Metadata for a SQL database.
+ */
+ SQLMetadata: {
+ connection: string;
+ database: string;
+ schema: string;
+ /** @enum {unknown} */
+ type: "sql-metadata";
+ };
/**
* SQLTableListPreview
* @description Preview of a list of tables in a schema.
@@ -4353,6 +4364,7 @@ export interface components {
SQLTableListPreview: {
/** @default null */
error?: string | null;
+ metadata: components["schemas"]["SQLMetadata"];
/** @enum {unknown} */
op: "sql-table-list-preview";
request_id: string;
@@ -4366,6 +4378,7 @@ export interface components {
SQLTablePreview: {
/** @default null */
error?: string | null;
+ metadata: components["schemas"]["SQLMetadata"];
/** @enum {unknown} */
op: "sql-table-preview";
request_id: string;
diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.py
index 89e127b477..940f7ee247 100644
--- a/tests/_runtime/test_runtime_datasets.py
+++ b/tests/_runtime/test_runtime_datasets.py
@@ -7,6 +7,7 @@ import pytest
from marimo._dependencies.dependencies import DependencyManager
from marimo._messaging.ops import (
DataSourceConnections,
+ SQLMetadata,
SQLTableListPreview,
SQLTablePreview,
ValidateSQLResult,
@@ -99,7 +100,12 @@ class TestPreviewSQLTable:
]
assert preview_sql_table_results == [
SQLTablePreview(
- request_id=RequestId("0"), table=None, error="Engine not found"
+ request_id=RequestId("0"),
+ table=None,
+ error="Engine not found",
+ metadata=SQLMetadata(
+ connection=DUCKDB_CONN, database="test", schema="test"
+ ),
)
]
@@ -126,7 +132,14 @@ class TestPreviewSQLTable:
op for op in stream.operations if isinstance(op, SQLTablePreview)
]
assert preview_sql_table_results == [
- SQLTablePreview(request_id=RequestId("0"), table=None, error=None)
+ SQLTablePreview(
+ request_id=RequestId("0"),
+ table=None,
+ error=None,
+ metadata=SQLMetadata(
+ connection=DUCKDB_CONN, database="test", schema="test"
+ ),
+ )
]
async def test_query_engine(
@@ -156,6 +169,9 @@ class TestPreviewSQLTable:
request_id=RequestId("0"),
table=None,
error="Connection does not support catalog operations",
+ metadata=SQLMetadata(
+ connection=SQLITE_CONN, database="test", schema="test"
+ ),
)
]
@@ -182,7 +198,12 @@ class TestPreviewSQLTableList:
]
assert preview_sql_table_list_results == [
SQLTableListPreview(
- request_id=RequestId("0"), tables=[], error="Engine not found"
+ request_id=RequestId("0"),
+ tables=[],
+ error="Engine not found",
+ metadata=SQLMetadata(
+ connection=DUCKDB_CONN, database="test", schema="test"
+ ),
)
]
@@ -211,7 +232,12 @@ class TestPreviewSQLTableList:
]
assert preview_sql_table_list_results == [
SQLTableListPreview(
- request_id=RequestId("0"), tables=[], error=None
+ request_id=RequestId("0"),
+ tables=[],
+ error=None,
+ metadata=SQLMetadata(
+ connection=DUCKDB_CONN, database="test", schema="test"
+ ),
)
]
@@ -243,6 +269,9 @@ class TestPreviewSQLTableList:
request_id=RequestId("0"),
tables=[],
error="Connection does not support catalog operations",
+ metadata=SQLMetadata(
+ connection=SQLITE_CONN, database="test", schema="test"
+ ),
)
]
diff --git a/tests/_server/session/test_session_view.py b/tests/_server/session/test_session_view.py
index 82b129df5f..e36dba5e0e 100644
--- a/tests/_server/session/test_session_view.py
+++ b/tests/_server/session/test_session_view.py
@@ -5,7 +5,7 @@ from typing import Any
from unittest.mock import patch
from marimo._ast.cell import RuntimeStateType
-from marimo._data.models import DataTable, DataTableColumn
+from marimo._data.models import Database, DataTable, DataTableColumn, Schema
from marimo._messaging.cell_output import CellChannel, CellOutput
from marimo._messaging.msgspec_encoder import asdict as serialize
from marimo._messaging.ops import (
@@ -15,6 +15,9 @@ from marimo._messaging.ops import (
DataSourceConnections,
InstallingPackageAlert,
SendUIElementMessage,
+ SQLMetadata,
+ SQLTableListPreview,
+ SQLTablePreview,
StartupLogs,
UpdateCellCodes,
UpdateCellIdsRequest,
@@ -32,7 +35,7 @@ from marimo._runtime.requests import (
)
from marimo._server.session.session_view import SessionView
from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE
-from marimo._types.ids import CellId_t, WidgetModelId
+from marimo._types.ids import CellId_t, RequestId, VariableName, WidgetModelId
from marimo._utils.parse_dataclass import parse_raw
cell_id = CellId_t("cell_1")
@@ -558,6 +561,125 @@ def test_add_data_source_connections(session_view: SessionView) -> None:
assert INTERNAL_DUCKDB_ENGINE in session_view_names
+def test_add_sql_table_previews() -> None:
+ session_view = SessionView()
+
+ # Add initial connections
+ session_view.add_raw_operation(
+ serialize_kernel_message(
+ DataSourceConnections(
+ connections=[
+ DataSourceConnection(
+ source="duckdb",
+ name="connection1",
+ dialect="duckdb",
+ display_name="duckdb (connection1)",
+ databases=[
+ Database(
+ name="db1",
+ dialect="duckdb",
+ schemas=[
+ Schema(
+ name="db1",
+ tables=[
+ DataTable(
+ name="table1",
+ source_type="connection",
+ source="db1",
+ columns=[],
+ num_rows=0,
+ num_columns=0,
+ variable_name=None,
+ )
+ ],
+ )
+ ],
+ )
+ ],
+ )
+ ],
+ )
+ )
+ )
+
+ session_view_connections = session_view.data_connectors.connections
+ assert session_view_connections[0].databases[0].schemas[0].tables == [
+ DataTable(
+ source_type="connection",
+ source="db1",
+ name="table1",
+ num_rows=0,
+ num_columns=0,
+ variable_name=None,
+ columns=[],
+ )
+ ]
+
+ session_view.add_raw_operation(
+ serialize_kernel_message(
+ SQLTablePreview(
+ metadata=SQLMetadata(
+ connection="connection1", database="db1", schema="db1"
+ ),
+ request_id=RequestId("request_id"),
+ table=DataTable(
+ name="table1",
+ source_type="connection",
+ source="db1",
+ num_rows=10, # Updated
+ num_columns=0,
+ variable_name=None,
+ columns=[],
+ ),
+ )
+ )
+ )
+ session_view_connections = session_view.data_connectors.connections
+ assert (
+ session_view_connections[0].databases[0].schemas[0].tables[0].num_rows
+ == 10
+ )
+
+ # Add sql table preview list
+ session_view.add_raw_operation(
+ serialize_kernel_message(
+ SQLTableListPreview(
+ metadata=SQLMetadata(
+ connection="connection1", database="db1", schema="db1"
+ ),
+ request_id=RequestId("request_id"),
+ tables=[
+ DataTable(
+ name="table2",
+ source_type="connection",
+ source="db1",
+ num_rows=20,
+ num_columns=10,
+ variable_name=VariableName("var"),
+ columns=[],
+ )
+ ],
+ )
+ )
+ )
+
+ assert session_view_connections[0].databases[0].schemas[0].tables == [
+ DataTable(
+ source_type="connection",
+ source="db1",
+ name="table2",
+ num_rows=20,
+ num_columns=10,
+ variable_name=VariableName("var"),
+ columns=[],
+ engine=None,
+ type="table",
+ primary_keys=None,
+ indexes=None,
+ )
+ ]
+
+
def test_add_cell_op(session_view: SessionView) -> None:
session_view.add_raw_operation(
serialize_kernel_message(
diff --git a/tests/_sql/test_connection_utils.py b/tests/_sql/test_connection_utils.py
new file mode 100644
index 0000000000..225d3f17b7
--- /dev/null
+++ b/tests/_sql/test_connection_utils.py
@@ -0,0 +1,489 @@
+# Copyright 2025 Marimo. All rights reserved.
+
+import time
+from typing import Callable
+
+import pytest
+
+from marimo._data.models import (
+ Database,
+ DataSourceConnection,
+ DataTable,
+ DataTableColumn,
+ Schema,
+)
+from marimo._messaging.ops import SQLMetadata
+from marimo._sql.connection_utils import (
+ update_table_in_connection,
+ update_table_list_in_connection,
+)
+
+TIME_THRESHOLD_SECONDS = 0.1
+
+
+def create_test_table(name: str = "test_table") -> DataTable:
+ """Create a test DataTable."""
+ return DataTable(
+ source_type="connection",
+ source="postgres",
+ name=name,
+ num_rows=100,
+ num_columns=3,
+ variable_name=None,
+ columns=[
+ DataTableColumn(
+ name="id",
+ type="integer",
+ external_type="INT",
+ sample_values=[1, 2, 3],
+ ),
+ DataTableColumn(
+ name="name",
+ type="string",
+ external_type="VARCHAR",
+ sample_values=["Alice", "Bob", "Charlie"],
+ ),
+ DataTableColumn(
+ name="age",
+ type="integer",
+ external_type="INT",
+ sample_values=[25, 30, 35],
+ ),
+ ],
+ )
+
+
+def create_test_connections(
+ num_connections: int = 1,
+ num_databases_per_conn: int = 1,
+ num_schemas_per_db: int = 1,
+ num_tables_per_schema: int = 1,
+) -> list[DataSourceConnection]:
+ """Create test data source connections with a hierarchical structure."""
+ connections = []
+ for conn_idx in range(num_connections):
+ databases = []
+ for db_idx in range(num_databases_per_conn):
+ schemas = []
+ for schema_idx in range(num_schemas_per_db):
+ tables = [
+ create_test_table(f"table_{table_idx}")
+ for table_idx in range(num_tables_per_schema)
+ ]
+ schemas.append(
+ Schema(name=f"schema_{schema_idx}", tables=tables)
+ )
+ databases.append(
+ Database(
+ name=f"database_{db_idx}",
+ dialect="postgresql",
+ schemas=schemas,
+ )
+ )
+ connections.append(
+ DataSourceConnection(
+ source="postgres",
+ dialect="postgresql",
+ name=f"connection_{conn_idx}",
+ display_name=f"PostgreSQL (connection_{conn_idx})",
+ databases=databases,
+ )
+ )
+ return connections
+
+
+class TestUpdateTableInConnection:
+ """Tests for update_table_in_connection function."""
+
+ def test_update_existing_table(self) -> None:
+ """Test updating an existing table in the hierarchy."""
+ connections = create_test_connections(
+ num_connections=2,
+ num_databases_per_conn=2,
+ num_schemas_per_db=2,
+ num_tables_per_schema=3,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_1",
+ database="database_1",
+ schema="schema_1",
+ )
+
+ # Create updated table
+ updated_table = create_test_table("table_2")
+ updated_table.num_rows = 500 # Changed value
+
+ update_table_in_connection(connections, sql_metadata, updated_table)
+
+ # Verify the update
+ target_schema = connections[1].databases[1].schemas[1]
+ updated = target_schema.tables[2]
+ assert updated.name == "table_2"
+ assert updated.num_rows == 500
+
+ def test_update_nonexistent_connection(self) -> None:
+ """Test updating a table in a non-existent connection."""
+ connections = create_test_connections()
+
+ sql_metadata = SQLMetadata(
+ connection="nonexistent",
+ database="database_0",
+ schema="schema_0",
+ )
+
+ updated_table = create_test_table()
+ original_table = connections[0].databases[0].schemas[0].tables[0]
+ original_rows = original_table.num_rows
+
+ update_table_in_connection(connections, sql_metadata, updated_table)
+
+ # Verify nothing changed
+ assert original_table.num_rows == original_rows
+
+ def test_update_nonexistent_database(self) -> None:
+ """Test updating a table in a non-existent database."""
+ connections = create_test_connections()
+
+ sql_metadata = SQLMetadata(
+ connection="connection_0",
+ database="nonexistent",
+ schema="schema_0",
+ )
+
+ updated_table = create_test_table()
+ original_table = connections[0].databases[0].schemas[0].tables[0]
+ original_rows = original_table.num_rows
+
+ update_table_in_connection(connections, sql_metadata, updated_table)
+
+ # Verify nothing changed
+ assert original_table.num_rows == original_rows
+
+ def test_update_nonexistent_schema(self) -> None:
+ """Test updating a table in a non-existent schema."""
+ connections = create_test_connections()
+
+ sql_metadata = SQLMetadata(
+ connection="connection_0",
+ database="database_0",
+ schema="nonexistent",
+ )
+
+ updated_table = create_test_table()
+ original_table = connections[0].databases[0].schemas[0].tables[0]
+ original_rows = original_table.num_rows
+
+ update_table_in_connection(connections, sql_metadata, updated_table)
+
+ # Verify nothing changed
+ assert original_table.num_rows == original_rows
+
+ def test_update_nonexistent_table(self) -> None:
+ """Test updating a non-existent table."""
+ connections = create_test_connections(num_tables_per_schema=2)
+
+ sql_metadata = SQLMetadata(
+ connection="connection_0",
+ database="database_0",
+ schema="schema_0",
+ )
+
+ updated_table = create_test_table("nonexistent_table")
+ original_tables = connections[0].databases[0].schemas[0].tables[:]
+
+ update_table_in_connection(connections, sql_metadata, updated_table)
+
+ # Verify nothing changed
+ assert connections[0].databases[0].schemas[0].tables == original_tables
+
+
+class TestUpdateTableListInConnection:
+ """Tests for update_table_list_in_connection function."""
+
+ def test_update_table_list(self) -> None:
+ """Test updating a table list in the hierarchy."""
+ connections = create_test_connections(num_tables_per_schema=3)
+
+ sql_metadata = SQLMetadata(
+ connection="connection_0",
+ database="database_0",
+ schema="schema_0",
+ )
+
+ # Create new table list
+ new_tables = [create_test_table(f"new_table_{i}") for i in range(5)]
+
+ update_table_list_in_connection(connections, sql_metadata, new_tables)
+
+ # Verify the update
+ target_schema = connections[0].databases[0].schemas[0]
+ assert len(target_schema.tables) == 5
+ assert target_schema.tables[0].name == "new_table_0"
+ assert target_schema.tables[4].name == "new_table_4"
+
+ def test_update_table_list_nonexistent_connection(self) -> None:
+ """Test updating a table list in a non-existent connection."""
+ connections = create_test_connections(num_tables_per_schema=3)
+
+ sql_metadata = SQLMetadata(
+ connection="nonexistent",
+ database="database_0",
+ schema="schema_0",
+ )
+
+ new_tables = [create_test_table(f"new_table_{i}") for i in range(5)]
+ original_count = len(connections[0].databases[0].schemas[0].tables)
+
+ update_table_list_in_connection(connections, sql_metadata, new_tables)
+
+ # Verify nothing changed
+ assert (
+ len(connections[0].databases[0].schemas[0].tables)
+ == original_count
+ )
+
+
+class TestPerformance:
+ """Performance tests for connection utils."""
+
+ def _measure_performance(
+ self,
+ func: Callable[[], None],
+ iterations: int = 1000,
+ ) -> tuple[float, float]:
+ """Measure average and total execution time.
+
+ Returns:
+ Tuple of (average_time_ms, total_time_ms)
+ """
+ start = time.perf_counter()
+ for _ in range(iterations):
+ func()
+ total_time = (time.perf_counter() - start) * 1000 # Convert to ms
+ avg_time = total_time / iterations
+ return avg_time, total_time
+
+ def test_performance_small_hierarchy(self) -> None:
+ """Benchmark with small hierarchy (typical real-world case)."""
+ # 2 connections, 3 databases each, 5 schemas each, 20 tables each
+ connections = create_test_connections(
+ num_connections=2,
+ num_databases_per_conn=3,
+ num_schemas_per_db=5,
+ num_tables_per_schema=20,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_1",
+ database="database_2",
+ schema="schema_4",
+ )
+
+ updated_table = create_test_table("table_19")
+ updated_table.num_rows = 999
+
+ def update_func() -> None:
+ update_table_in_connection(
+ connections, sql_metadata, updated_table
+ )
+
+ avg_time, total_time = self._measure_performance(
+ update_func, iterations=1000
+ )
+
+ print(
+ f"\nSmall hierarchy (2x3x5x20 = 600 tables): "
+ f"{avg_time:.4f}ms avg, {total_time:.2f}ms total"
+ )
+
+ # Assert reasonable performance: < 0.1ms per operation
+ assert avg_time < 0.1, f"Performance too slow: {avg_time:.4f}ms"
+
+ def test_performance_medium_hierarchy(self) -> None:
+ """Benchmark with medium hierarchy."""
+ # 5 connections, 5 databases each, 10 schemas each, 50 tables each
+ connections = create_test_connections(
+ num_connections=5,
+ num_databases_per_conn=5,
+ num_schemas_per_db=10,
+ num_tables_per_schema=50,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_4",
+ database="database_4",
+ schema="schema_9",
+ )
+
+ updated_table = create_test_table("table_49")
+ updated_table.num_rows = 999
+
+ def update_func() -> None:
+ update_table_in_connection(
+ connections, sql_metadata, updated_table
+ )
+
+ avg_time, total_time = self._measure_performance(
+ update_func, iterations=1000
+ )
+
+ print(
+ f"\nMedium hierarchy (5x5x10x50 = 12,500 tables): "
+ f"{avg_time:.4f}ms avg, {total_time:.2f}ms total"
+ )
+
+ # Assert reasonable performance: < 0.5ms per operation
+ assert avg_time < TIME_THRESHOLD_SECONDS, (
+ f"Performance too slow: {avg_time:.4f}ms"
+ )
+
+ def test_performance_large_hierarchy(self) -> None:
+ """Benchmark with large hierarchy (stress test)."""
+ # 10 connections, 10 databases each, 10 schemas each, 100 tables each
+ connections = create_test_connections(
+ num_connections=10,
+ num_databases_per_conn=10,
+ num_schemas_per_db=10,
+ num_tables_per_schema=100,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_9",
+ database="database_9",
+ schema="schema_9",
+ )
+
+ updated_table = create_test_table("table_99")
+ updated_table.num_rows = 999
+
+ def update_func() -> None:
+ update_table_in_connection(
+ connections, sql_metadata, updated_table
+ )
+
+ avg_time, total_time = self._measure_performance(
+ update_func, iterations=1000
+ )
+
+ print(
+ f"\nLarge hierarchy (10x10x10x100 = 100,000 tables): "
+ f"{avg_time:.4f}ms avg, {total_time:.2f}ms total"
+ )
+
+ # Assert reasonable performance: < 1ms per operation
+ assert avg_time < TIME_THRESHOLD_SECONDS, (
+ f"Performance too slow: {avg_time:.4f}ms"
+ )
+
+ def test_performance_worst_case_early_exit(self) -> None:
+ """Benchmark best case: early exit on first connection."""
+ connections = create_test_connections(
+ num_connections=10,
+ num_databases_per_conn=10,
+ num_schemas_per_db=10,
+ num_tables_per_schema=100,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_0",
+ database="database_0",
+ schema="schema_0",
+ )
+
+ updated_table = create_test_table("table_0")
+ updated_table.num_rows = 999
+
+ def update_func() -> None:
+ update_table_in_connection(
+ connections, sql_metadata, updated_table
+ )
+
+ avg_time, total_time = self._measure_performance(
+ update_func, iterations=1000
+ )
+
+ print(
+ f"\nBest case (early exit): "
+ f"{avg_time:.4f}ms avg, {total_time:.2f}ms total"
+ )
+
+ # Early exit should be very fast
+ assert avg_time < 0.1, f"Performance too slow: {avg_time:.4f}ms"
+
+ def test_performance_update_table_list(self) -> None:
+ """Benchmark update_table_list_in_connection."""
+ connections = create_test_connections(
+ num_connections=5,
+ num_databases_per_conn=5,
+ num_schemas_per_db=10,
+ num_tables_per_schema=50,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_4",
+ database="database_4",
+ schema="schema_9",
+ )
+
+ new_tables = [create_test_table(f"new_table_{i}") for i in range(50)]
+
+ def update_func() -> None:
+ update_table_list_in_connection(
+ connections, sql_metadata, new_tables
+ )
+
+ avg_time, total_time = self._measure_performance(
+ update_func, iterations=1000
+ )
+
+ print(
+ f"\nUpdate table list (5x5x10x50 = 12,500 tables): "
+ f"{avg_time:.4f}ms avg, {total_time:.2f}ms total"
+ )
+
+ # Should be fast since we're just replacing a list reference
+ assert avg_time < TIME_THRESHOLD_SECONDS, (
+ f"Performance too slow: {avg_time:.4f}ms"
+ )
+
+ @pytest.mark.skip(
+ reason="Only run manually for detailed profiling analysis"
+ )
+ def test_profile_with_cprofile(self) -> None:
+ """Profile the function with cProfile for detailed analysis."""
+ import cProfile
+ import pstats
+
+ connections = create_test_connections(
+ num_connections=10,
+ num_databases_per_conn=10,
+ num_schemas_per_db=10,
+ num_tables_per_schema=100,
+ )
+
+ sql_metadata = SQLMetadata(
+ connection="connection_9",
+ database="database_9",
+ schema="schema_9",
+ )
+
+ updated_table = create_test_table("table_99")
+
+ profiler = cProfile.Profile()
+ profiler.enable()
+
+ for _ in range(10000):
+ update_table_in_connection(
+ connections, sql_metadata, updated_table
+ )
+
+ profiler.disable()
+
+ stats = pstats.Stats(profiler)
+ stats.strip_dirs()
+ stats.sort_stats("cumulative")
+ print("\n" + "=" * 80)
+ print("cProfile Results (top 20):")
+ print("=" * 80)
+ stats.print_stats(20)