A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: support Dremio ADBC data source browsing (#9694)
Matt Corley90369026e90e
diff --git a/frontend/src/components/databases/display.tsx b/frontend/src/components/databases/display.tsxindex 8b74c5e735..75c33be31b 100644--- a/frontend/src/components/databases/display.tsx+++ b/frontend/src/components/databases/display.tsx@@ -53,6 +53,8 @@ export function dbDisplayName(name: string) {return "MongoDB";case "iceberg":return "Apache Iceberg";+ case "dremio":+ return "Dremio";default:return name;}diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.tsindex e259658f6d..b3e7c35121 100644--- a/frontend/src/components/datasources/__tests__/utils.test.ts+++ b/frontend/src/components/datasources/__tests__/utils.test.ts@@ -313,6 +313,26 @@ describe("sqlCode", () => {);});+ it("should preserve dots inside quoted schema names", () => {+ const sqlTableContext: SQLTableContext = {+ engine: "postgres",+ schema: "analytics.events",+ defaultSchema: "public",+ defaultDatabase: "mydb",+ database: "remote",+ dialect: "postgres",+ };++ const result = sqlCode({+ table: mockTable,+ columnName: mockColumn.name,+ sqlTableContext,+ });+ expect(result).toBe(+ '_df = mo.sql(f"""\nSELECT "email" FROM "remote"."analytics.events"."users" LIMIT 100\n""", engine=postgres)',+ );+ });+it("should not quote * column name", () => {const sqlTableContext: SQLTableContext = {engine: "postgres",@@ -334,6 +354,68 @@ describe("sqlCode", () => {});});+ describe("Dremio dialect", () => {+ it("should quote reserved column names and table path parts", () => {+ const sqlTableContext: SQLTableContext = {+ engine: "dremio_conn",+ schema: "operations",+ defaultSchema: "",+ defaultDatabase: "",+ database: "lakehouse",+ dialect: "dremio",+ };++ const result = sqlCode({+ table: { ...mockTable, name: "shipments" as const },+ columnName: "order",+ sqlTableContext,+ });+ expect(result).toBe(+ '_df = mo.sql(f"""\nSELECT "order" FROM "lakehouse"."operations"."shipments" LIMIT 100\n""", engine=dremio_conn)',+ );+ });++ it("should not quote * column name", () => {+ const sqlTableContext: SQLTableContext = {+ engine: "dremio_conn",+ schema: "operations",+ defaultSchema: "",+ defaultDatabase: "",+ database: "lakehouse",+ dialect: "dremio",+ };++ const result = sqlCode({+ table: { ...mockTable, name: "customers" as const },+ columnName: "*",+ sqlTableContext,+ });+ expect(result).toBe(+ '_df = mo.sql(f"""\nSELECT * FROM "lakehouse"."operations"."customers" LIMIT 100\n""", engine=dremio_conn)',+ );+ });++ it("should preserve dots inside quoted schema names", () => {+ const sqlTableContext: SQLTableContext = {+ engine: "dremio_conn",+ schema: "samples.dremio.com",+ defaultSchema: "",+ defaultDatabase: "",+ database: "Samples",+ dialect: "dremio",+ };++ const result = sqlCode({+ table: { ...mockTable, name: "airlines" as const },+ columnName: "*",+ sqlTableContext,+ });+ expect(result).toBe(+ '_df = mo.sql(f"""\nSELECT * FROM "Samples"."samples.dremio.com"."airlines" LIMIT 100\n""", engine=dremio_conn)',+ );+ });+ });+describe("fallback behavior", () => {it("should use default formatter for unknown dialect", () => {const sqlTableContext: SQLTableContext = {diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.tsindex dbd9d86785..113d892805 100644--- a/frontend/src/components/datasources/utils.ts+++ b/frontend/src/components/datasources/utils.ts@@ -15,9 +15,9 @@ export function isSchemaless(schemaName: string) {interface SqlCodeFormatter {/**- * Format the table name based on dialect-specific rules+ * Format the table path based on dialect-specific rules*/- formatTableName: (tableName: string) => string;+ formatTablePath: (tablePath: string[]) => string;/*** Format the SELECT clause*/@@ -25,7 +25,7 @@ interface SqlCodeFormatter {}const defaultFormatter: SqlCodeFormatter = {- formatTableName: (tableName: string) => tableName,+ formatTablePath: (tablePath: string[]) => tablePath.join("."),formatSelectClause: (columnName: string, tableName: string) =>`SELECT ${columnName} FROM ${tableName} LIMIT 100`,};@@ -41,7 +41,8 @@ function getFormatter(dialect: string): SqlCodeFormatter {const quote = BigQueryDialect.spec.identifierQuotes;return {// BigQuery uses backticks for identifiers- formatTableName: (tableName: string) => `${quote}${tableName}${quote}`,+ formatTablePath: (tablePath: string[]) =>+ `${quote}${tablePath.join(".")}${quote}`,formatSelectClause: defaultFormatter.formatSelectClause,};}@@ -49,7 +50,7 @@ function getFormatter(dialect: string): SqlCodeFormatter {case "sqlserver":case "microsoft sql server":return {- formatTableName: defaultFormatter.formatTableName,+ formatTablePath: defaultFormatter.formatTablePath,formatSelectClause: (columnName: string, tableName: string) =>`SELECT TOP 100 ${columnName} FROM ${tableName}`,};@@ -57,12 +58,11 @@ function getFormatter(dialect: string): SqlCodeFormatter {case "postgres":case "postgresql":case "duckdb":+ case "dremio":// Quote column and table names to avoid raising errors on weird charactersreturn {- formatTableName: (tableName: string) => {- const parts = tableName.split(".");- return parts.map((part) => `"${part}"`).join(".");- },+ formatTablePath: (tablePath: string[]) =>+ tablePath.map((part) => `"${part}"`).join("."),formatSelectClause: (columnName: string, tableName: string) =>`SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,};@@ -114,26 +114,27 @@ export function sqlCode({database,dialect,} = sqlTableContext;- let tableName = table.name;+ const tablePath = [table.name];// Set the fully qualified table name based on schema and databaseif (isSchemaless(schema)) {- tableName =- database === defaultDatabase ? tableName : `${database}.${tableName}`;+ if (database !== defaultDatabase) {+ tablePath.unshift(database);+ }} else {// Include schema if it's not the default schemaif (schema !== defaultSchema) {- tableName = `${schema}.${tableName}`;+ tablePath.unshift(schema);}// Include database if it's not the default databaseif (database !== defaultDatabase) {- tableName = `${database}.${tableName}`;+ tablePath.unshift(database);}}const formatter = getFormatter(dialect);- const formattedTableName = formatter.formatTableName(tableName);+ const formattedTableName = formatter.formatTablePath(tablePath);const selectClause = formatter.formatSelectClause(columnName,formattedTableName,diff --git a/frontend/src/core/codemirror/format.ts b/frontend/src/core/codemirror/format.tsindex 800ecf21a3..297b02f267 100644--- a/frontend/src/core/codemirror/format.ts+++ b/frontend/src/core/codemirror/format.ts@@ -205,6 +205,7 @@ async function getSqlFormatterDialect(case "mongodb":case "timescaledb":case "datafusion":+ case "dremio":return sql;case "databricks":return spark;diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.tsindex 435f1173f4..3a35405d2e 100644--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts@@ -410,6 +410,7 @@ function connectionNameToParserDialect(case "spark":case "databricks":case "datafusion":+ case "dremio":Logger.debug("Unsupported dialect", { dialect });return null;default:diff --git a/frontend/src/core/codemirror/language/languages/sql/utils.ts b/frontend/src/core/codemirror/language/languages/sql/utils.tsindex 19763a402d..2189936cf2 100644--- a/frontend/src/core/codemirror/language/languages/sql/utils.ts+++ b/frontend/src/core/codemirror/language/languages/sql/utils.ts@@ -52,6 +52,7 @@ const KNOWN_DIALECTS_ARRAY = ["databricks","datafusion","microsoft sql server",+ "dremio",] as const;const KNOWN_DIALECTS: ReadonlySet<string> = new Set(KNOWN_DIALECTS_ARRAY);type KnownDialect = (typeof KNOWN_DIALECTS_ARRAY)[number];@@ -115,6 +116,7 @@ export function guessDialect(case "spark":case "databricks":case "datafusion":+ case "dremio":Logger.debug("Unsupported dialect", { dialect });return ModifiedStandardSQL;default:diff --git a/marimo/_sql/engines/adbc.py b/marimo/_sql/engines/adbc.pyindex c7c7427a9a..37321bed18 100644--- a/marimo/_sql/engines/adbc.py+++ b/marimo/_sql/engines/adbc.py@@ -30,6 +30,22 @@ AdbcGetObjectsDepth = Literal["all", "catalogs", "db_schemas", "tables", "columns"]+ARROW_INTEGER_TYPES = frozenset(+ f"{prefix}{bits}"+ for prefix in ("int", "uint")+ for bits in ("8", "16", "32", "64")+)+ARROW_STRING_TYPES = frozenset(+ (+ "binary",+ "binary_view",+ "large_binary",+ "large_string",+ "string",+ "string_view",+ )+)+class AdbcDbApiCursor(Protocol):description: Any@@ -108,10 +124,22 @@ def _adbc_info_to_dialect(*, info: dict[str | int, Any]) -> str:def _schema_field_to_data_type(external_type: str) -> DataType:"""Map an Arrow-like dtype string to marimo DataType."""- t = external_type.lower()- if "bool" in t:+ t = external_type.strip().lower()+ if t.startswith(+ (+ "array<",+ "fixed_size_list<",+ "large_list<",+ "list<",+ "map<",+ "struct<",+ "union<",+ )+ ) or t.endswith("[]"):+ return "unknown"+ if t == "bool":return "boolean"- if "int" in t or "uint" in t:+ if t in ARROW_INTEGER_TYPES:return "integer"if "float" in t or "double" in t or "decimal" in t:return "number"@@ -121,7 +149,10 @@ def _schema_field_to_data_type(external_type: str) -> DataType:return "date"if t.startswith("time") or " time" in t:return "time"- return "string"+ # Arrow renders fixed-size binary with its width, e.g. fixed_size_binary[16].+ if t in ARROW_STRING_TYPES or t.startswith("fixed_size_binary["):+ return "string"+ return "unknown"class AdbcConnectionCatalog:diff --git a/tests/_sql/test_adbc.py b/tests/_sql/test_adbc.pyindex aa307fa7eb..1b2bb191ce 100644--- a/tests/_sql/test_adbc.py+++ b/tests/_sql/test_adbc.py@@ -11,6 +11,7 @@ from marimo._sql.engines.adbc import (AdbcConnectionCatalog,AdbcDBAPIEngine,_adbc_info_to_dialect,+ _schema_field_to_data_type,)from marimo._sql.get_engines import get_engines_from_variablesfrom marimo._types.ids import VariableName@@ -229,6 +230,7 @@ def test_adbc_info_to_dialect() -> None:_adbc_info_to_dialect(info={"vendor_name": "Microsoft SQL Server"})== "microsoft sql server")+ assert _adbc_info_to_dialect(info={"vendor_name": "Dremio"}) == "dremio"# Missing/blank/non-string vendor_name falls back to "sql".assert _adbc_info_to_dialect(info={"vendor_name": " "}) == "sql"@@ -237,6 +239,48 @@ def test_adbc_info_to_dialect() -> None:assert _adbc_info_to_dialect(info={}) == "sql"+ "external_type",+ [+ "list<$data$: bool not null>",+ "list<$data$: int64 not null>",+ "large_list<$data$: double>",+ "fixed_size_list<item: string>[3]",+ "array<int32>",+ "int64[]",+ " struct<id: int64, enabled: bool> ",+ "struct<id: int64, enabled: bool>",+ "map<string, int64>",+ ],+)+def test_schema_field_to_data_type_returns_unknown_for_nested_types(+ external_type: str,+) -> None:+ assert _schema_field_to_data_type(external_type) == "unknown"+++ ("external_type", "data_type"),+ [+ ("int8", "integer"),+ ("int64", "integer"),+ ("uint32", "integer"),+ ("uint64", "integer"),+ ("string", "string"),+ ("large_string", "string"),+ ("binary", "string"),+ ("fixed_size_binary[16]", "string"),+ ("interval", "unknown"),+ ("dictionary<values=string, indices=int8, ordered=0>", "unknown"),+ ("bool", "boolean"),+ ],+)+def test_schema_field_to_data_type_handles_scalar_types(+ external_type: str, data_type: str+) -> None:+ assert _schema_field_to_data_type(external_type) == data_type++def test_get_engines_from_variables_prefers_adbc_dbapi_engine() -> None:conn = FakeAdbcDbApiConnection(cursor=FakeAdbcDbApiCursor(description=None),