A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add databricks to connection form (#6528)
Shahmir Varqhaea007de882f2
diff --git a/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap b/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snapindex 497b88b4fa..f2491ca76e 100644--- a/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap+++ b/frontend/src/components/editor/database/__tests__/__snapshots__/as-code.test.ts.snap@@ -28,6 +28,42 @@ engine = clickhouse_connect.get_client()"`;+exports[`generateDatabaseCode > basic connections > databricks 1`] = `+"import os+import sqlalchemy++_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")+DATABASE_URL = f"databricks://token:{_access_token}@{_server_hostname}?http_path={_http_path}"+engine = sqlalchemy.create_engine(DATABASE_URL)"+`;++exports[`generateDatabaseCode > basic connections > databricks with catalog and schema 1`] = `+"import os+import sqlalchemy++_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")+DATABASE_URL = f"databricks://token:{_access_token}@{_server_hostname}?http_path={_http_path}&catalog=my_catalog&schema=my_schema"+engine = sqlalchemy.create_engine(DATABASE_URL)"+`;++exports[`generateDatabaseCode > basic connections > databricks with ibis 1`] = `+"import ibis+import os++_access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "my_access_token")+_server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", "localhost")+_http_path = os.environ.get("DATABRICKS_HTTP_PATH", "http://localhost:8080")+engine = ibis.databricks.connect(+ server_hostname=_server_hostname,+ http_path=_http_path,+ access_token=_access_token+)"+`;+exports[`generateDatabaseCode > basic connections > datafusion 1`] = `"from datafusion import SessionContextimport ibisdiff --git a/frontend/src/components/editor/database/__tests__/as-code.test.ts b/frontend/src/components/editor/database/__tests__/as-code.test.tsindex 84ae1f4688..49c84df5d6 100644--- a/frontend/src/components/editor/database/__tests__/as-code.test.ts+++ b/frontend/src/components/editor/database/__tests__/as-code.test.ts@@ -192,6 +192,22 @@ describe("generateDatabaseCode", () => {},};+ const databricksConnection: DatabaseConnection = {+ type: "databricks",+ access_token: "my_access_token",+ server_hostname: "localhost",+ http_path: "http://localhost:8080",+ };++ const databricksWithCatalogSchema: DatabaseConnection = {+ type: "databricks",+ access_token: "my_access_token",+ server_hostname: "localhost",+ http_path: "http://localhost:8080",+ catalog: "my_catalog",+ schema: "my_schema",+ };+describe("basic connections", () => {const testCases: Array<[string, DatabaseConnection, ConnectionLibrary]> = [["postgres with SQLModel", basePostgres, "sqlmodel"],@@ -218,6 +234,13 @@ describe("generateDatabaseCode", () => {["pyspark with session", pysparkConnSession, "ibis"],["redshift with DB credentials", redshiftDBConnection, "redshift"],["redshift with IAM credentials", redshiftIAMConnection, "redshift"],+ ["databricks", databricksConnection, "sqlalchemy"],+ [+ "databricks with catalog and schema",+ databricksWithCatalogSchema,+ "sqlalchemy",+ ],+ ["databricks with ibis", databricksConnection, "ibis"],];it.each(testCases)("%s", (name, connection, orm) => {diff --git a/frontend/src/components/editor/database/add-database-form.tsx b/frontend/src/components/editor/database/add-database-form.tsxindex 52e4768e95..74d976ef73 100644--- a/frontend/src/components/editor/database/add-database-form.tsx+++ b/frontend/src/components/editor/database/add-database-form.tsx@@ -39,6 +39,7 @@ import {ChdbConnectionSchema,ClickhouseConnectionSchema,type DatabaseConnection,+ DatabricksConnectionSchema,DataFusionConnectionSchema,DuckDBConnectionSchema,IcebergConnectionSchema,@@ -210,6 +211,16 @@ const DATABASES = [preferred: "redshift",},},+ {+ name: "Databricks",+ schema: DatabricksConnectionSchema,+ color: "#c41e0c",+ logo: "databricks",+ connectionLibraries: {+ libraries: ["sqlalchemy", "sqlmodel", "ibis"],+ preferred: "sqlalchemy",+ },+ },] satisfies ConnectionSchema[];const DATA_CATALOGS = [diff --git a/frontend/src/components/editor/database/as-code.ts b/frontend/src/components/editor/database/as-code.tsindex 3616bf0d2c..7b1f473aab 100644--- a/frontend/src/components/editor/database/as-code.ts+++ b/frontend/src/components/editor/database/as-code.ts@@ -14,7 +14,8 @@ export type ConnectionLibrary =| "pyiceberg"| "ibis"| "motherduck"- | "redshift";+ | "redshift"+ | "databricks";export const ConnectionDisplayNames: Record<ConnectionLibrary, string> = {sqlmodel: "SQLModel",@@ -26,6 +27,7 @@ export const ConnectionDisplayNames: Record<ConnectionLibrary, string> = {ibis: "Ibis",motherduck: "MotherDuck",redshift: "Redshift",+ databricks: "Databricks",};abstract class CodeGenerator<T extends DatabaseConnection["type"]> {@@ -55,6 +57,9 @@ abstract class CodeGenerator<T extends DatabaseConnection["type"]> {case "duckdb":imports.add("import duckdb");break;+ case "ibis":+ imports.add("import ibis");+ break;}return imports;}@@ -138,10 +143,38 @@ class SecretContainer {return `{${value}}`;}+ /**+ * Generate a password variable for connection strings, supporting inline values,+ * environment variable lookups, and f-string formatting.+ * @param {string|undefined} password - Fallback password value.+ * @param {string} passwordPlaceholder - Environment variable name.+ * @param {boolean} inFString - If true, wrap for Python f-string.+ * @param {string} [variableName] - Variable name (default: "password").+ * @returns {string}+ *+ * @example+ * `printPassword("token123", "DATABRICKS_ACCESS_TOKEN", true, "access_token")`+ *+ * Returns:+ * ```python+ * _access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "token123")+ * f"db://sample:{_access_token}@sample.com"+ * ```+ *+ * @example+ * `printPassword("token123", "DATABRICKS_ACCESS_TOKEN", false, "access_token")`+ *+ * Returns:+ * ```python+ * access_token = os.environ.get("DATABRICKS_ACCESS_TOKEN", "token123")+ * f"db://sample:[email protected]"+ * ```+ */printPassword(password: string | undefined,passwordPlaceholder: string,inFString: boolean,+ variableName?: string,): string {// Inline passwords should use printInFString, otherwise use printconst printMethod = inFString@@ -149,8 +182,8 @@ class SecretContainer {: this.print.bind(this);return isSecret(password)- ? printMethod("password", password)- : printMethod("password", passwordPlaceholder, password);+ ? printMethod(variableName || "password", password)+ : printMethod(variableName || "password", passwordPlaceholder, password);}getSecrets(): Record<string, string> {@@ -557,7 +590,7 @@ class PyIcebergGenerator extends CodeGenerator<"iceberg"> {class DataFusionGenerator extends CodeGenerator<"datafusion"> {generateImports(): string[] {// To trigger installation of ibis-datafusion- return ["import ibis", "from datafusion import SessionContext"];+ return ["from datafusion import SessionContext"];}generateConnectionCode(): string {@@ -578,7 +611,7 @@ class DataFusionGenerator extends CodeGenerator<"datafusion"> {class PySparkGenerator extends CodeGenerator<"pyspark"> {generateImports(): string[] {- return ["import ibis", "from pyspark.sql import SparkSession"];+ return ["from pyspark.sql import SparkSession"];}generateConnectionCode(): string {@@ -668,6 +701,70 @@ ${formatUrlParams(params, (inner) => ` ${inner}`)},}}+class DatabricksGenerator extends CodeGenerator<"databricks"> {+ generateImports(): string[] {+ return [];+ }++ generateConnectionCode(): string {+ const useFString = this.orm !== "ibis";++ const accessToken = this.secrets.printPassword(+ this.connection.access_token,+ "DATABRICKS_ACCESS_TOKEN",+ useFString,+ "access_token",+ );++ const serverHostname = this.secrets.printPassword(+ this.connection.server_hostname,+ "DATABRICKS_SERVER_HOSTNAME",+ useFString,+ "server_hostname",+ );++ const httpPath = this.secrets.printPassword(+ this.connection.http_path,+ "DATABRICKS_HTTP_PATH",+ useFString,+ "http_path",+ );++ const catalog = this.connection.catalog+ ? this.secrets.printInFString("catalog", this.connection.catalog)+ : undefined;+ const schema = this.connection.schema+ ? this.secrets.printInFString("schema", this.connection.schema)+ : undefined;++ let BASE_URL = `databricks://token:${accessToken}@${serverHostname}?http_path=${httpPath}`;+ if (catalog) {+ BASE_URL += `&catalog=${catalog}`;+ }+ if (schema) {+ BASE_URL += `&schema=${schema}`;+ }++ if (this.orm === "ibis") {+ const catalogParam = catalog ? `\n catalog=${catalog},` : "";+ const schemaParam = schema ? `\n schema=${schema},` : "";++ return dedent(`+ engine = ibis.databricks.connect(+ server_hostname=${serverHostname},+ http_path=${httpPath},${catalogParam}${schemaParam}+ access_token=${accessToken}+ )+ `);+ }++ return dedent(`+ DATABASE_URL = f"${BASE_URL}"+ engine = ${this.orm}.create_engine(DATABASE_URL)+ `);+ }+}+class CodeGeneratorFactory {public secrets = new SecretContainer();@@ -706,6 +803,8 @@ class CodeGeneratorFactory {return new PySparkGenerator(connection, orm, this.secrets);case "redshift":return new RedshiftGenerator(connection, orm, this.secrets);+ case "databricks":+ return new DatabricksGenerator(connection, orm, this.secrets);default:assertNever(connection);}diff --git a/frontend/src/components/editor/database/schemas.ts b/frontend/src/components/editor/database/schemas.tsindex 1f8f074c29..19a6f2df1f 100644--- a/frontend/src/components/editor/database/schemas.ts+++ b/frontend/src/components/editor/database/schemas.ts@@ -16,18 +16,16 @@ function passwordField() {);}-function tokenField() {- return z- .string()- .optional()- .describe(- FieldOptions.of({- label: "Token",- inputType: "password",- placeholder: "token",- optionRegex: ".*token.*",- }),- );+function tokenField(label?: string, required?: boolean) {+ const field = z.string().describe(+ FieldOptions.of({+ label: label || "Token",+ inputType: "password",+ placeholder: "token",+ optionRegex: ".*token.*",+ }),+ );+ return required ? field.nonempty() : field.optional();}function warehouseNameField() {@@ -43,20 +41,22 @@ function warehouseNameField() {);}-function uriField() {- return z+function uriField(label?: string, required?: boolean) {+ const field = z.string()- .optional()- .describe(FieldOptions.of({ label: "URI", optionRegex: ".*uri.*" }));+ .describe(+ FieldOptions.of({ label: label || "URI", optionRegex: ".*uri.*" }),+ );+ return required ? field.nonempty() : field.optional();}-function hostField() {+function hostField(label?: string) {return z.string().nonempty().describe(FieldOptions.of({- label: "Host",+ label: label || "Host",placeholder: "localhost",optionRegex: ".*host.*",}),@@ -464,6 +464,23 @@ export const RedshiftConnectionSchema = z}).describe(FieldOptions.of({ direction: "two-columns" }));+export const DatabricksConnectionSchema = z+ .object({+ type: z.literal("databricks"),+ access_token: tokenField("Access Token", true),+ server_hostname: hostField("Server Hostname"),+ http_path: uriField("HTTP Path", true),+ catalog: z+ .string()+ .optional()+ .describe(FieldOptions.of({ label: "Catalog" })),+ schema: z+ .string()+ .optional()+ .describe(FieldOptions.of({ label: "Schema" })),+ })+ .describe(FieldOptions.of({ direction: "two-columns" }));+export const DatabaseConnectionSchema = z.discriminatedUnion("type", [PostgresConnectionSchema,MySQLConnectionSchema,@@ -480,6 +497,7 @@ export const DatabaseConnectionSchema = z.discriminatedUnion("type", [DataFusionConnectionSchema,PySparkConnectionSchema,RedshiftConnectionSchema,+ DatabricksConnectionSchema,]);export type DatabaseConnection = z.infer<typeof DatabaseConnectionSchema>;