Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

add databricks to connection form (#6528)

Shahmir Varqhaea007de882f2
diff --git a/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap
index 497b88b4fa..f2491ca76e 100644
--- a/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap
+++ b/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap
@@ -28,6 +28,42 @@ engine = clickhouse_connect.get_client(
)"
`;
+exports[`generateDatabaseCode > basic connections > databricks 1`] = `
+"import os
+import sqlalchemy
+
+_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")
+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")
+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")
+DATABASE_URL = f"databricks://token:{_access_token}@{_server_hostname}?http_path={_http_path}"
+engine = sqlalchemy.create_engine(DATABASE_URL)"
+`;
+
+exports[`generateDatabaseCode > basic connections > databricks with catalog and schema 1`] = `
+"import os
+import sqlalchemy
+
+_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")
+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")
+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")
+DATABASE_URL = f"databricks://token:{_access_token}@{_server_hostname}?http_path={_http_path}&catalog=my_catalog&schema=my_schema"
+engine = sqlalchemy.create_engine(DATABASE_URL)"
+`;
+
+exports[`generateDatabaseCode > basic connections > databricks with ibis 1`] = `
+"import ibis
+import os
+
+_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")
+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")
+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")
+engine = ibis.databricks.connect(
+ server_hostname=_server_hostname,
+ http_path=_http_path,
+ access_token=_access_token
+)"
+`;
+
exports[`generateDatabaseCode > basic connections > datafusion 1`] = `
"from datafusion import SessionContext
import ibis
diff --git a/frontend/src/components/editor/database/__tests__/as-code.test.ts b/frontend/src/components/editor/database/__tests__/as-code.test.ts
index 84ae1f4688..49c84df5d6 100644
--- a/frontend/src/components/editor/database/__tests__/as-code.test.ts
+++ b/frontend/src/components/editor/database/__tests__/as-code.test.ts
@@ -192,6 +192,22 @@ describe("generateDatabaseCode", () => {
},
};
+ const databricksConnection: DatabaseConnection = {
+ type: "databricks",
+ access_token: "my_access_token",
+ server_hostname: "localhost",
+ http_path: "http://localhost:8080",
+ };
+
+ const databricksWithCatalogSchema: DatabaseConnection = {
+ type: "databricks",
+ access_token: "my_access_token",
+ server_hostname: "localhost",
+ http_path: "http://localhost:8080",
+ catalog: "my_catalog",
+ schema: "my_schema",
+ };
+
describe("basic connections", () => {
const testCases: Array<[string, DatabaseConnection, ConnectionLibrary]> = [
["postgres with SQLModel", basePostgres, "sqlmodel"],
@@ -218,6 +234,13 @@ describe("generateDatabaseCode", () => {
["pyspark with session", pysparkConnSession, "ibis"],
["redshift with DB credentials", redshiftDBConnection, "redshift"],
["redshift with IAM credentials", redshiftIAMConnection, "redshift"],
+ ["databricks", databricksConnection, "sqlalchemy"],
+ [
+ "databricks with catalog and schema",
+ databricksWithCatalogSchema,
+ "sqlalchemy",
+ ],
+ ["databricks with ibis", databricksConnection, "ibis"],
];
it.each(testCases)("%s", (name, connection, orm) => {
diff --git a/frontend/src/components/editor/database/add-database-form.tsx b/frontend/src/components/editor/database/add-database-form.tsx
index 52e4768e95..74d976ef73 100644
--- a/frontend/src/components/editor/database/add-database-form.tsx
+++ b/frontend/src/components/editor/database/add-database-form.tsx
@@ -39,6 +39,7 @@ import {
ChdbConnectionSchema,
ClickhouseConnectionSchema,
type DatabaseConnection,
+ DatabricksConnectionSchema,
DataFusionConnectionSchema,
DuckDBConnectionSchema,
IcebergConnectionSchema,
@@ -210,6 +211,16 @@ const DATABASES = [
preferred: "redshift",
},
},
+ {
+ name: "Databricks",
+ schema: DatabricksConnectionSchema,
+ color: "#c41e0c",
+ logo: "databricks",
+ connectionLibraries: {
+ libraries: ["sqlalchemy", "sqlmodel", "ibis"],
+ preferred: "sqlalchemy",
+ },
+ },
] satisfies ConnectionSchema[];
const DATA_CATALOGS = [
diff --git a/frontend/src/components/editor/database/as-code.ts b/frontend/src/components/editor/database/as-code.ts
index 3616bf0d2c..7b1f473aab 100644
--- a/frontend/src/components/editor/database/as-code.ts
+++ b/frontend/src/components/editor/database/as-code.ts
@@ -14,7 +14,8 @@ export type ConnectionLibrary =
| "pyiceberg"
| "ibis"
| "motherduck"
- | "redshift";
+ | "redshift"
+ | "databricks";
export const ConnectionDisplayNames: Record<ConnectionLibrary, string> = {
sqlmodel: "SQLModel",
@@ -26,6 +27,7 @@ export const ConnectionDisplayNames: Record<ConnectionLibrary, string> = {
ibis: "Ibis",
motherduck: "MotherDuck",
redshift: "Redshift",
+ databricks: "Databricks",
};
abstract class CodeGenerator<T extends DatabaseConnection["type"]> {
@@ -55,6 +57,9 @@ abstract class CodeGenerator<T extends DatabaseConnection["type"]> {
case "duckdb":
imports.add("import duckdb");
break;
+ case "ibis":
+ imports.add("import ibis");
+ break;
}
return imports;
}
@@ -138,10 +143,38 @@ class SecretContainer {
return `{${value}}`;
}
+ /**
+ * Generate a password variable for connection strings, supporting inline values,
+ * environment variable lookups, and f-string formatting.
+ * @param {string|undefined} password - Fallback password value.
+ * @param {string} passwordPlaceholder - Environment variable name.
+ * @param {boolean} inFString - If true, wrap for Python f-string.
+ * @param {string} [variableName] - Variable name (default: "password").
+ * @returns {string}
+ *
+ * @example
+ * `printPassword("token123", "DATABRICKS_ACCESS_TOKEN", true, "access_token")`
+ *
+ * Returns:
+ * ```python
+ * _access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "token123")
+ * f"db://sample:{_access_token}@sample.com"
+ * ```
+ *
+ * @example
+ * `printPassword("token123", "DATABRICKS_ACCESS_TOKEN", false, "access_token")`
+ *
+ * Returns:
+ * ```python
+ * access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "token123")
+ * f"db://sample:[email protected]"
+ * ```
+ */
printPassword(
password: string | undefined,
passwordPlaceholder: string,
inFString: boolean,
+ variableName?: string,
): string {
// Inline passwords should use printInFString, otherwise use print
const printMethod = inFString
@@ -149,8 +182,8 @@ class SecretContainer {
: this.print.bind(this);
return isSecret(password)
- ? printMethod("password", password)
- : printMethod("password", passwordPlaceholder, password);
+ ? printMethod(variableName || "password", password)
+ : printMethod(variableName || "password", passwordPlaceholder, password);
}
getSecrets(): Record<string, string> {
@@ -557,7 +590,7 @@ class PyIcebergGenerator extends CodeGenerator<"iceberg"> {
class DataFusionGenerator extends CodeGenerator<"datafusion"> {
generateImports(): string[] {
// To trigger installation of ibis-datafusion
- return ["import ibis", "from datafusion import SessionContext"];
+ return ["from datafusion import SessionContext"];
}
generateConnectionCode(): string {
@@ -578,7 +611,7 @@ class DataFusionGenerator extends CodeGenerator<"datafusion"> {
class PySparkGenerator extends CodeGenerator<"pyspark"> {
generateImports(): string[] {
- return ["import ibis", "from pyspark.sql import SparkSession"];
+ return ["from pyspark.sql import SparkSession"];
}
generateConnectionCode(): string {
@@ -668,6 +701,70 @@ ${formatUrlParams(params, (inner) => ` ${inner}`)},
}
}
+class DatabricksGenerator extends CodeGenerator<"databricks"> {
+ generateImports(): string[] {
+ return [];
+ }
+
+ generateConnectionCode(): string {
+ const useFString = this.orm !== "ibis";
+
+ const accessToken = this.secrets.printPassword(
+ this.connection.access_token,
+ "DATABRICKS_ACCESS_TOKEN",
+ useFString,
+ "access_token",
+ );
+
+ const serverHostname = this.secrets.printPassword(
+ this.connection.server_hostname,
+ "DATABRICKS_SERVER_HOSTNAME",
+ useFString,
+ "server_hostname",
+ );
+
+ const httpPath = this.secrets.printPassword(
+ this.connection.http_path,
+ "DATABRICKS_HTTP_PATH",
+ useFString,
+ "http_path",
+ );
+
+ const catalog = this.connection.catalog
+ ? this.secrets.printInFString("catalog", this.connection.catalog)
+ : undefined;
+ const schema = this.connection.schema
+ ? this.secrets.printInFString("schema", this.connection.schema)
+ : undefined;
+
+ let BASE_URL = `databricks://token:${accessToken}@${serverHostname}?http_path=${httpPath}`;
+ if (catalog) {
+ BASE_URL += `&catalog=${catalog}`;
+ }
+ if (schema) {
+ BASE_URL += `&schema=${schema}`;
+ }
+
+ if (this.orm === "ibis") {
+ const catalogParam = catalog ? `\n catalog=${catalog},` : "";
+ const schemaParam = schema ? `\n schema=${schema},` : "";
+
+ return dedent(`
+ engine = ibis.databricks.connect(
+ server_hostname=${serverHostname},
+ http_path=${httpPath},${catalogParam}${schemaParam}
+ access_token=${accessToken}
+ )
+ `);
+ }
+
+ return dedent(`
+ DATABASE_URL = f"${BASE_URL}"
+ engine = ${this.orm}.create_engine(DATABASE_URL)
+ `);
+ }
+}
+
class CodeGeneratorFactory {
public secrets = new SecretContainer();
@@ -706,6 +803,8 @@ class CodeGeneratorFactory {
return new PySparkGenerator(connection, orm, this.secrets);
case "redshift":
return new RedshiftGenerator(connection, orm, this.secrets);
+ case "databricks":
+ return new DatabricksGenerator(connection, orm, this.secrets);
default:
assertNever(connection);
}
diff --git a/frontend/src/components/editor/database/schemas.ts b/frontend/src/components/editor/database/schemas.ts
index 1f8f074c29..19a6f2df1f 100644
--- a/frontend/src/components/editor/database/schemas.ts
+++ b/frontend/src/components/editor/database/schemas.ts
@@ -16,18 +16,16 @@ function passwordField() {
);
}
-function tokenField() {
- return z
- .string()
- .optional()
- .describe(
- FieldOptions.of({
- label: "Token",
- inputType: "password",
- placeholder: "token",
- optionRegex: ".*token.*",
- }),
- );
+function tokenField(label?: string, required?: boolean) {
+ const field = z.string().describe(
+ FieldOptions.of({
+ label: label || "Token",
+ inputType: "password",
+ placeholder: "token",
+ optionRegex: ".*token.*",
+ }),
+ );
+ return required ? field.nonempty() : field.optional();
}
function warehouseNameField() {
@@ -43,20 +41,22 @@ function warehouseNameField() {
);
}
-function uriField() {
- return z
+function uriField(label?: string, required?: boolean) {
+ const field = z
.string()
- .optional()
- .describe(FieldOptions.of({ label: "URI", optionRegex: ".*uri.*" }));
+ .describe(
+ FieldOptions.of({ label: label || "URI", optionRegex: ".*uri.*" }),
+ );
+ return required ? field.nonempty() : field.optional();
}
-function hostField() {
+function hostField(label?: string) {
return z
.string()
.nonempty()
.describe(
FieldOptions.of({
- label: "Host",
+ label: label || "Host",
placeholder: "localhost",
optionRegex: ".*host.*",
}),
@@ -464,6 +464,23 @@ export const RedshiftConnectionSchema = z
})
.describe(FieldOptions.of({ direction: "two-columns" }));
+export const DatabricksConnectionSchema = z
+ .object({
+ type: z.literal("databricks"),
+ access_token: tokenField("Access Token", true),
+ server_hostname: hostField("Server Hostname"),
+ http_path: uriField("HTTP Path", true),
+ catalog: z
+ .string()
+ .optional()
+ .describe(FieldOptions.of({ label: "Catalog" })),
+ schema: z
+ .string()
+ .optional()
+ .describe(FieldOptions.of({ label: "Schema" })),
+ })
+ .describe(FieldOptions.of({ direction: "two-columns" }));
+
export const DatabaseConnectionSchema = z.discriminatedUnion("type", [
PostgresConnectionSchema,
MySQLConnectionSchema,
@@ -480,6 +497,7 @@ export const DatabaseConnectionSchema = z.discriminatedUnion("type", [
DataFusionConnectionSchema,
PySparkConnectionSchema,
RedshiftConnectionSchema,
+ DatabricksConnectionSchema,
]);
export type DatabaseConnection = z.infer<typeof DatabaseConnectionSchema>;