Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat(data): add Databricks SQL warehouse discovery plugin (#10556)

Myles Scolnickdf8dbed044af
diff --git a/docs/guides/working_with_data/sql.md b/docs/guides/working_with_data/sql.md
index 8717c71497..3303d7f933 100644
--- a/docs/guides/working_with_data/sql.md
+++ b/docs/guides/working_with_data/sql.md
@@ -226,6 +226,7 @@ Click a suggestion to insert a ready-to-run cell.
| PostgreSQL | `PGHOST`, `PGUSER`, `PGDATABASE` (plus `PGPORT`, `PGPASSWORD`) |
| MySQL | `MYSQL_HOST`, `MYSQL_USER`, `MYSQL_DATABASE`, and `MYSQL_PASSWORD`/`MYSQL_PWD` (plus `MYSQL_TCP_PORT`) |
| Trino | `TRINO_HOST`, `TRINO_USER`, `TRINO_CATALOG` (plus `TRINO_PORT`, `TRINO_PASSWORD`, `TRINO_SCHEMA`) |
+| Databricks SQL warehouse | `DATABRICKS_SQL_WAREHOUSE_ID` (plus `DATABRICKS_CONFIG_PROFILE`) |
| PySpark (Spark Connect) | `SPARK_REMOTE` |
| PyIceberg catalogs | `PYICEBERG_CATALOG__<name>__*` environment variables, or catalogs resolved from a `.pyiceberg.yaml` file |
diff --git a/marimo/_data/data_source_discovery/plugins/__init__.py b/marimo/_data/data_source_discovery/plugins/__init__.py
index bbc279ecb5..7e8910d8fd 100644
--- a/marimo/_data/data_source_discovery/plugins/__init__.py
+++ b/marimo/_data/data_source_discovery/plugins/__init__.py
@@ -1,5 +1,8 @@
# Copyright 2026 Marimo. All rights reserved.
from marimo._data.data_source_discovery.plugins.aws import AWS_PLUGIN
+from marimo._data.data_source_discovery.plugins.databricks import (
+ DATABRICKS_PLUGIN,
+)
from marimo._data.data_source_discovery.plugins.huggingface import (
HUGGINGFACE_PLUGIN,
)
@@ -17,6 +20,7 @@ DEFAULT_DISCOVERY_PLUGINS: tuple[DiscoveryPlugin, ...] = (
MYSQL_PLUGIN,
AWS_PLUGIN,
TRINO_PLUGIN,
+ DATABRICKS_PLUGIN,
PYSPARK_PLUGIN,
PYICEBERG_PLUGIN,
HUGGINGFACE_PLUGIN,
diff --git a/marimo/_data/data_source_discovery/plugins/databricks.py b/marimo/_data/data_source_discovery/plugins/databricks.py
new file mode 100644
index 0000000000..f33182c0a0
--- /dev/null
+++ b/marimo/_data/data_source_discovery/plugins/databricks.py
@@ -0,0 +1,70 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+from marimo._data.data_source_discovery.helpers import (
+ ENVIRONMENT_ORIGIN,
+ environment_variable,
+ has_value,
+)
+from marimo._data.data_source_discovery.models import DetectedDataSource
+from marimo._data.data_source_discovery.types import (
+ DiscoveryContext,
+ DiscoveryPlugin,
+)
+
+
+def discover(context: DiscoveryContext) -> list[DetectedDataSource]:
+ environment = context.environment
+ if not has_value(environment, "DATABRICKS_SQL_WAREHOUSE_ID"):
+ return []
+
+ configuration = [
+ environment_variable("SQL warehouse", "DATABRICKS_SQL_WAREHOUSE_ID")
+ ]
+ has_profile = has_value(environment, "DATABRICKS_CONFIG_PROFILE")
+ if has_profile:
+ configuration.append(
+ environment_variable("Profile", "DATABRICKS_CONFIG_PROFILE")
+ )
+
+ workspace_client = (
+ "workspace = WorkspaceClient(\n"
+ ' profile=os.environ["DATABRICKS_CONFIG_PROFILE"]\n'
+ ")"
+ if has_profile
+ else "workspace = WorkspaceClient()"
+ )
+ code = f"""\
+import os
+from urllib.parse import urlsplit
+
+from databricks import sql
+from databricks.sdk import WorkspaceClient
+
+{workspace_client}
+connection = sql.connect(
+ server_hostname=urlsplit(workspace.config.host).hostname,
+ http_path=(
+ "/sql/1.0/warehouses/"
+ + os.environ["DATABRICKS_SQL_WAREHOUSE_ID"]
+ ),
+ access_token=workspace.config.authenticate()[
+ "Authorization"
+ ].removeprefix("Bearer "),
+)"""
+
+ return [
+ DetectedDataSource(
+ id="databricks-sql-warehouse-environment",
+ integration="databricks",
+ category="database",
+ display_name="Databricks SQL warehouse",
+ confidence="high",
+ origins=(ENVIRONMENT_ORIGIN,),
+ configuration=tuple(configuration),
+ code=code,
+ )
+ ]
+
+
+DATABRICKS_PLUGIN = DiscoveryPlugin(id="databricks", discover=discover)
diff --git a/tests/_data/data_source_discovery/test_databricks.py b/tests/_data/data_source_discovery/test_databricks.py
new file mode 100644
index 0000000000..754da0da3d
--- /dev/null
+++ b/tests/_data/data_source_discovery/test_databricks.py
@@ -0,0 +1,178 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import ast
+import sys
+from types import ModuleType, SimpleNamespace
+from typing import TYPE_CHECKING
+
+import msgspec
+from inline_snapshot import snapshot
+
+from marimo._data.data_source_discovery.plugins.databricks import discover
+from marimo._data.data_source_discovery.types import DiscoveryContext
+
+if TYPE_CHECKING:
+ import pytest
+
+
+def test_discovers_databricks_sql_warehouse() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-warehouse-id",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins == snapshot(
+ [
+ {
+ "id": "databricks-sql-warehouse-environment",
+ "integration": "databricks",
+ "category": "database",
+ "displayName": "Databricks SQL warehouse",
+ "confidence": "high",
+ "origins": [
+ {
+ "type": "environment",
+ "label": "Kernel environment",
+ }
+ ],
+ "configuration": [
+ {
+ "field": "SQL warehouse",
+ "value": {
+ "kind": "environment-variable",
+ "name": "DATABRICKS_SQL_WAREHOUSE_ID",
+ },
+ }
+ ],
+ "code": """\
+import os
+from urllib.parse import urlsplit
+
+from databricks import sql
+from databricks.sdk import WorkspaceClient
+
+workspace = WorkspaceClient()
+connection = sql.connect(
+ server_hostname=urlsplit(workspace.config.host).hostname,
+ http_path=(
+ "/sql/1.0/warehouses/"
+ + os.environ["DATABRICKS_SQL_WAREHOUSE_ID"]
+ ),
+ access_token=workspace.config.authenticate()[
+ "Authorization"
+ ].removeprefix("Bearer "),
+)""",
+ }
+ ]
+ )
+ ast.parse(detected[0].code)
+ assert "secret-warehouse-id" not in repr(builtins)
+
+
+def test_uses_optional_databricks_profile() -> None:
+ detected = discover(
+ DiscoveryContext(
+ environment={
+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-warehouse-id",
+ "DATABRICKS_CONFIG_PROFILE": "secret-profile",
+ }
+ )
+ )
+
+ builtins = msgspec.json.decode(msgspec.json.encode(detected))
+ assert builtins[0]["configuration"] == snapshot(
+ [
+ {
+ "field": "SQL warehouse",
+ "value": {
+ "kind": "environment-variable",
+ "name": "DATABRICKS_SQL_WAREHOUSE_ID",
+ },
+ },
+ {
+ "field": "Profile",
+ "value": {
+ "kind": "environment-variable",
+ "name": "DATABRICKS_CONFIG_PROFILE",
+ },
+ },
+ ]
+ )
+ assert (
+ """\
+workspace = WorkspaceClient(
+ profile=os.environ["DATABRICKS_CONFIG_PROFILE"]
+)"""
+ in detected[0].code
+ )
+ ast.parse(detected[0].code)
+ assert "secret-" not in repr(builtins)
+
+
+def test_ignores_missing_or_empty_warehouse_id() -> None:
+ assert discover(DiscoveryContext(environment={})) == snapshot([])
+ assert discover(
+ DiscoveryContext(environment={"DATABRICKS_SQL_WAREHOUSE_ID": ""})
+ ) == snapshot([])
+
+
+def test_generated_code_connects_with_unified_authentication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ profiles: list[str | None] = []
+ connection_arguments: list[dict[str, object]] = []
+ expected_connection = object()
+
+ class WorkspaceClient:
+ def __init__(self, *, profile: str | None = None) -> None:
+ profiles.append(profile)
+ self.config = SimpleNamespace(
+ host="https://workspace.cloud.databricks.com",
+ authenticate=lambda: {
+ "Authorization": "Bearer secret-access-token"
+ },
+ )
+
+ def connect(**kwargs: object) -> object:
+ connection_arguments.append(kwargs)
+ return expected_connection
+
+ databricks = ModuleType("databricks")
+ databricks_sql = ModuleType("databricks.sql")
+ databricks_sql.connect = connect # type: ignore[attr-defined]
+ databricks.sql = databricks_sql # type: ignore[attr-defined]
+ databricks_sdk = ModuleType("databricks.sdk")
+ databricks_sdk.WorkspaceClient = WorkspaceClient # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, "databricks", databricks)
+ monkeypatch.setitem(sys.modules, "databricks.sql", databricks_sql)
+ monkeypatch.setitem(sys.modules, "databricks.sdk", databricks_sdk)
+ monkeypatch.setenv("DATABRICKS_SQL_WAREHOUSE_ID", "warehouse-id")
+ monkeypatch.setenv("DATABRICKS_CONFIG_PROFILE", "analytics")
+
+ source = discover(
+ DiscoveryContext(
+ environment={
+ "DATABRICKS_SQL_WAREHOUSE_ID": "warehouse-id",
+ "DATABRICKS_CONFIG_PROFILE": "analytics",
+ }
+ )
+ )[0]
+ namespace: dict[str, object] = {}
+ exec(compile(source.code, "<databricks-discovery>", "exec"), namespace)
+
+ assert profiles == snapshot(["analytics"])
+ assert connection_arguments == snapshot(
+ [
+ {
+ "server_hostname": "workspace.cloud.databricks.com",
+ "http_path": "/sql/1.0/warehouses/warehouse-id",
+ "access_token": "secret-access-token",
+ }
+ ]
+ )
+ assert namespace["connection"] is expected_connection
diff --git a/tests/_data/data_source_discovery/test_discover.py b/tests/_data/data_source_discovery/test_discover.py
index 810e5119d8..7b8b46dc07 100644
--- a/tests/_data/data_source_discovery/test_discover.py
+++ b/tests/_data/data_source_discovery/test_discover.py
@@ -89,6 +89,8 @@ def test_default_plugins_emit_valid_secret_free_suggestions() -> None:
"TRINO_PASSWORD": "secret-trino-password",
"TRINO_CATALOG": "secret-trino-catalog",
"TRINO_SCHEMA": "secret-trino-schema",
+ "DATABRICKS_SQL_WAREHOUSE_ID": "secret-databricks-warehouse",
+ "DATABRICKS_CONFIG_PROFILE": "secret-databricks-profile",
"SPARK_REMOTE": "sc://secret-spark-host:61234",
"PYICEBERG_CATALOG__PROD__URI": "https://secret-iceberg.invalid",
}
@@ -111,6 +113,7 @@ def test_default_plugins_emit_valid_secret_free_suggestions() -> None:
"mysql",
"aws",
"trino",
+ "databricks",
"pyspark",
"pyiceberg",
"pyiceberg",