A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat(data): add Databricks SQL warehouse discovery plugin (#10556)
Myles Scolnickdf8dbed044af
diff --git a/docs/guides/working_with_data/sql.md b/docs/guides/working_with_data/sql.mdindex 8717c71497..3303d7f933 100644--- a/docs/guides/working_with_data/sql.md+++ b/docs/guides/working_with_data/sql.md@@ -226,6 +226,7 @@ Click a suggestion to insert a ready-to-run cell.| PostgreSQL | `PGHOST`, `PGUSER`, `PGDATABASE` (plus `PGPORT`, `PGPASSWORD`) || MySQL | `MYSQL_HOST`, `MYSQL_USER`, `MYSQL_DATABASE`, and `MYSQL_PASSWORD`/`MYSQL_PWD` (plus `MYSQL_TCP_PORT`) || Trino | `TRINO_HOST`, `TRINO_USER`, `TRINO_CATALOG` (plus `TRINO_PORT`, `TRINO_PASSWORD`, `TRINO_SCHEMA`) |+| Databricks SQL warehouse | `DATABRICKS_SQL_WAREHOUSE_ID` (plus `DATABRICKS_CONFIG_PROFILE`) || PySpark (Spark Connect) | `SPARK_REMOTE` || PyIceberg catalogs | `PYICEBERG_CATALOG__<name>__*` environment variables, or catalogs resolved from a `.pyiceberg.yaml` file |diff --git a/marimo/_data/data_source_discovery/plugins/__init__.py b/marimo/_data/data_source_discovery/plugins/__init__.pyindex bbc279ecb5..7e8910d8fd 100644--- a/marimo/_data/data_source_discovery/plugins/__init__.py+++ b/marimo/_data/data_source_discovery/plugins/__init__.py@@ -1,5 +1,8 @@# Copyright 2026 Marimo. All rights reserved.from marimo._data.data_source_discovery.plugins.aws import AWS_PLUGIN+from marimo._data.data_source_discovery.plugins.databricks import (+ DATABRICKS_PLUGIN,+)from marimo._data.data_source_discovery.plugins.huggingface import (HUGGINGFACE_PLUGIN,)@@ -17,6 +20,7 @@ DEFAULT_DISCOVERY_PLUGINS: tuple[DiscoveryPlugin, ...] = (MYSQL_PLUGIN,AWS_PLUGIN,TRINO_PLUGIN,+ DATABRICKS_PLUGIN,PYSPARK_PLUGIN,PYICEBERG_PLUGIN,HUGGINGFACE_PLUGIN,diff --git a/marimo/_data/data_source_discovery/plugins/databricks.py b/marimo/_data/data_source_discovery/plugins/databricks.pynew file mode 100644index 0000000000..f33182c0a0--- /dev/null+++ b/marimo/_data/data_source_discovery/plugins/databricks.py@@ -0,0 +1,70 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from marimo._data.data_source_discovery.helpers import (+ ENVIRONMENT_ORIGIN,+ environment_variable,+ has_value,+)+from marimo._data.data_source_discovery.models import DetectedDataSource+from marimo._data.data_source_discovery.types import (+ DiscoveryContext,+ DiscoveryPlugin,+)+++def discover(context: DiscoveryContext) -> list[DetectedDataSource]:+ environment = context.environment+ if not has_value(environment, "DATABRICKS_SQL_WAREHOUSE_ID"):+ return []++ configuration = [+ environment_variable("SQL warehouse", "DATABRICKS_SQL_WAREHOUSE_ID")+ ]+ has_profile = has_value(environment, "DATABRICKS_CONFIG_PROFILE")+ if has_profile:+ configuration.append(+ environment_variable("Profile", "DATABRICKS_CONFIG_PROFILE")+ )++ workspace_client = (+ "workspace = WorkspaceClient(\n"+ ' profile=os.environ["DATABRICKS_CONFIG_PROFILE"]\n'+ ")"+ if has_profile+ else "workspace = WorkspaceClient()"+ )+ code = f"""\+import os+from urllib.parse import urlsplit++from databricks import sql+from databricks.sdk import WorkspaceClient++{workspace_client}+connection = sql.connect(+ server_hostname=urlsplit(workspace.config.host).hostname,+ http_path=(+ "/sql/1.0/warehouses/"+ + os.environ["DATABRICKS_SQL_WAREHOUSE_ID"]+ ),+ access_token=workspace.config.authenticate()[+ "Authorization"+ ].removeprefix("Bearer "),+)"""++ return [+ DetectedDataSource(+ id="databricks-sql-warehouse-environment",+ integration="databricks",+ category="database",+ display_name="Databricks SQL warehouse",+ confidence="high",+ origins=(ENVIRONMENT_ORIGIN,),+ configuration=tuple(configuration),+ code=code,+ )+ ]+++DATABRICKS_PLUGIN = DiscoveryPlugin(id="databricks", discover=discover)diff --git a/tests/_data/data_source_discovery/test_databricks.py b/tests/_data/data_source_discovery/test_databricks.pynew file mode 100644index 0000000000..754da0da3d--- /dev/null+++ b/tests/_data/data_source_discovery/test_databricks.py@@ -0,0 +1,178 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import ast+import sys+from types import ModuleType, SimpleNamespace+from typing import TYPE_CHECKING++import msgspec+from inline_snapshot import snapshot++from marimo._data.data_source_discovery.plugins.databricks import discover+from marimo._data.data_source_discovery.types import DiscoveryContext++if TYPE_CHECKING:+ import pytest+++def test_discovers_databricks_sql_warehouse() -> None:+ detected = discover(+ DiscoveryContext(+ environment={+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-warehouse-id",+ }+ )+ )++ builtins = msgspec.json.decode(msgspec.json.encode(detected))+ assert builtins == snapshot(+ [+ {+ "id": "databricks-sql-warehouse-environment",+ "integration": "databricks",+ "category": "database",+ "displayName": "Databricks SQL warehouse",+ "confidence": "high",+ "origins": [+ {+ "type": "environment",+ "label": "Kernel environment",+ }+ ],+ "configuration": [+ {+ "field": "SQL warehouse",+ "value": {+ "kind": "environment-variable",+ "name": "DATABRICKS_SQL_WAREHOUSE_ID",+ },+ }+ ],+ "code": """\+import os+from urllib.parse import urlsplit++from databricks import sql+from databricks.sdk import WorkspaceClient++workspace = WorkspaceClient()+connection = sql.connect(+ server_hostname=urlsplit(workspace.config.host).hostname,+ http_path=(+ "/sql/1.0/warehouses/"+ + os.environ["DATABRICKS_SQL_WAREHOUSE_ID"]+ ),+ access_token=workspace.config.authenticate()[+ "Authorization"+ ].removeprefix("Bearer "),+)""",+ }+ ]+ )+ ast.parse(detected[0].code)+ assert "secret-warehouse-id" not in repr(builtins)+++def test_uses_optional_databricks_profile() -> None:+ detected = discover(+ DiscoveryContext(+ environment={+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-warehouse-id",+ "DATABRICKS_CONFIG_PROFILE": "secret-profile",+ }+ )+ )++ builtins = msgspec.json.decode(msgspec.json.encode(detected))+ assert builtins[0]["configuration"] == snapshot(+ [+ {+ "field": "SQL warehouse",+ "value": {+ "kind": "environment-variable",+ "name": "DATABRICKS_SQL_WAREHOUSE_ID",+ },+ },+ {+ "field": "Profile",+ "value": {+ "kind": "environment-variable",+ "name": "DATABRICKS_CONFIG_PROFILE",+ },+ },+ ]+ )+ assert (+ """\+workspace = WorkspaceClient(+ profile=os.environ["DATABRICKS_CONFIG_PROFILE"]+)"""+ in detected[0].code+ )+ ast.parse(detected[0].code)+ assert "secret-" not in repr(builtins)+++def test_ignores_missing_or_empty_warehouse_id() -> None:+ assert discover(DiscoveryContext(environment={})) == snapshot([])+ assert discover(+ DiscoveryContext(environment={"DATABRICKS_SQL_WAREHOUSE_ID": ""})+ ) == snapshot([])+++def test_generated_code_connects_with_unified_authentication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ profiles: list[str | None] = []+ connection_arguments: list[dict[str, object]] = []+ expected_connection = object()++ class WorkspaceClient:+ def __init__(self, *, profile: str | None = None) -> None:+ profiles.append(profile)+ self.config = SimpleNamespace(+ host="https://workspace.cloud.databricks.com",+ authenticate=lambda: {+ "Authorization": "Bearer secret-access-token"+ },+ )++ def connect(**kwargs: object) -> object:+ connection_arguments.append(kwargs)+ return expected_connection++ databricks = ModuleType("databricks")+ databricks_sql = ModuleType("databricks.sql")+ databricks_sql.connect = connect # type: ignore[attr-defined]+ databricks.sql = databricks_sql # type: ignore[attr-defined]+ databricks_sdk = ModuleType("databricks.sdk")+ databricks_sdk.WorkspaceClient = WorkspaceClient # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, "databricks", databricks)+ monkeypatch.setitem(sys.modules, "databricks.sql", databricks_sql)+ monkeypatch.setitem(sys.modules, "databricks.sdk", databricks_sdk)+ monkeypatch.setenv("DATABRICKS_SQL_WAREHOUSE_ID", "warehouse-id")+ monkeypatch.setenv("DATABRICKS_CONFIG_PROFILE", "analytics")++ source = discover(+ DiscoveryContext(+ environment={+ "DATABRICKS_SQL_WAREHOUSE_ID": "warehouse-id",+ "DATABRICKS_CONFIG_PROFILE": "analytics",+ }+ )+ )[0]+ namespace: dict[str, object] = {}+ exec(compile(source.code, "<databricks-discovery>", "exec"), namespace)++ assert profiles == snapshot(["analytics"])+ assert connection_arguments == snapshot(+ [+ {+ "server_hostname": "workspace.cloud.databricks.com",+ "http_path": "/sql/1.0/warehouses/warehouse-id",+ "access_token": "secret-access-token",+ }+ ]+ )+ assert namespace["connection"] is expected_connectiondiff --git a/tests/_data/data_source_discovery/test_discover.py b/tests/_data/data_source_discovery/test_discover.pyindex 810e5119d8..7b8b46dc07 100644--- a/tests/_data/data_source_discovery/test_discover.py+++ b/tests/_data/data_source_discovery/test_discover.py@@ -89,6 +89,8 @@ def test_default_plugins_emit_valid_secret_free_suggestions() -> None:"TRINO_PASSWORD": "secret-trino-password","TRINO_CATALOG": "secret-trino-catalog","TRINO_SCHEMA": "secret-trino-schema",+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-databricks-warehouse",+ "DATABRICKS_CONFIG_PROFILE": "secret-databricks-profile","SPARK_REMOTE": "sc://secret-spark-host:61234","PYICEBERG_CATALOG__PROD__URI": "https://secret-iceberg.invalid",}@@ -111,6 +113,7 @@ def test_default_plugins_emit_valid_secret_free_suggestions() -> None:"mysql","aws","trino",+ "databricks","pyspark","pyiceberg","pyiceberg",