Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: support Dremio ADBC data source browsing (#9694)

Matt Corley90369026e90e
diff --git a/frontend/src/components/databases/display.tsx b/frontend/src/components/databases/display.tsx
index 8b74c5e735..75c33be31b 100644
--- a/frontend/src/components/databases/display.tsx
+++ b/frontend/src/components/databases/display.tsx
@@ -53,6 +53,8 @@ export function dbDisplayName(name: string) {
return "MongoDB";
case "iceberg":
return "Apache Iceberg";
+ case "dremio":
+ return "Dremio";
default:
return name;
}
diff --git a/frontend/src/components/datasources/__tests__/utils.test.ts b/frontend/src/components/datasources/__tests__/utils.test.ts
index e259658f6d..b3e7c35121 100644
--- a/frontend/src/components/datasources/__tests__/utils.test.ts
+++ b/frontend/src/components/datasources/__tests__/utils.test.ts
@@ -313,6 +313,26 @@ describe("sqlCode", () => {
);
});
+ it("should preserve dots inside quoted schema names", () => {
+ const sqlTableContext: SQLTableContext = {
+ engine: "postgres",
+ schema: "analytics.events",
+ defaultSchema: "public",
+ defaultDatabase: "mydb",
+ database: "remote",
+ dialect: "postgres",
+ };
+
+ const result = sqlCode({
+ table: mockTable,
+ columnName: mockColumn.name,
+ sqlTableContext,
+ });
+ expect(result).toBe(
+ '_df = mo.sql(f"""\nSELECT "email" FROM "remote"."analytics.events"."users" LIMIT 100\n""", engine=postgres)',
+ );
+ });
+
it("should not quote * column name", () => {
const sqlTableContext: SQLTableContext = {
engine: "postgres",
@@ -334,6 +354,68 @@ describe("sqlCode", () => {
});
});
+ describe("Dremio dialect", () => {
+ it("should quote reserved column names and table path parts", () => {
+ const sqlTableContext: SQLTableContext = {
+ engine: "dremio_conn",
+ schema: "operations",
+ defaultSchema: "",
+ defaultDatabase: "",
+ database: "lakehouse",
+ dialect: "dremio",
+ };
+
+ const result = sqlCode({
+ table: { ...mockTable, name: "shipments" as const },
+ columnName: "order",
+ sqlTableContext,
+ });
+ expect(result).toBe(
+ '_df = mo.sql(f"""\nSELECT "order" FROM "lakehouse"."operations"."shipments" LIMIT 100\n""", engine=dremio_conn)',
+ );
+ });
+
+ it("should not quote * column name", () => {
+ const sqlTableContext: SQLTableContext = {
+ engine: "dremio_conn",
+ schema: "operations",
+ defaultSchema: "",
+ defaultDatabase: "",
+ database: "lakehouse",
+ dialect: "dremio",
+ };
+
+ const result = sqlCode({
+ table: { ...mockTable, name: "customers" as const },
+ columnName: "*",
+ sqlTableContext,
+ });
+ expect(result).toBe(
+ '_df = mo.sql(f"""\nSELECT * FROM "lakehouse"."operations"."customers" LIMIT 100\n""", engine=dremio_conn)',
+ );
+ });
+
+ it("should preserve dots inside quoted schema names", () => {
+ const sqlTableContext: SQLTableContext = {
+ engine: "dremio_conn",
+ schema: "samples.dremio.com",
+ defaultSchema: "",
+ defaultDatabase: "",
+ database: "Samples",
+ dialect: "dremio",
+ };
+
+ const result = sqlCode({
+ table: { ...mockTable, name: "airlines" as const },
+ columnName: "*",
+ sqlTableContext,
+ });
+ expect(result).toBe(
+ '_df = mo.sql(f"""\nSELECT * FROM "Samples"."samples.dremio.com"."airlines" LIMIT 100\n""", engine=dremio_conn)',
+ );
+ });
+ });
+
describe("fallback behavior", () => {
it("should use default formatter for unknown dialect", () => {
const sqlTableContext: SQLTableContext = {
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.ts
index dbd9d86785..113d892805 100644
--- a/frontend/src/components/datasources/utils.ts
+++ b/frontend/src/components/datasources/utils.ts
@@ -15,9 +15,9 @@ export function isSchemaless(schemaName: string) {
interface SqlCodeFormatter {
/**
- * Format the table name based on dialect-specific rules
+ * Format the table path based on dialect-specific rules
*/
- formatTableName: (tableName: string) => string;
+ formatTablePath: (tablePath: string[]) => string;
/**
* Format the SELECT clause
*/
@@ -25,7 +25,7 @@ interface SqlCodeFormatter {
}
const defaultFormatter: SqlCodeFormatter = {
- formatTableName: (tableName: string) => tableName,
+ formatTablePath: (tablePath: string[]) => tablePath.join("."),
formatSelectClause: (columnName: string, tableName: string) =>
`SELECT ${columnName} FROM ${tableName} LIMIT 100`,
};
@@ -41,7 +41,8 @@ function getFormatter(dialect: string): SqlCodeFormatter {
const quote = BigQueryDialect.spec.identifierQuotes;
return {
// BigQuery uses backticks for identifiers
- formatTableName: (tableName: string) => `${quote}${tableName}${quote}`,
+ formatTablePath: (tablePath: string[]) =>
+ `${quote}${tablePath.join(".")}${quote}`,
formatSelectClause: defaultFormatter.formatSelectClause,
};
}
@@ -49,7 +50,7 @@ function getFormatter(dialect: string): SqlCodeFormatter {
case "sqlserver":
case "microsoft sql server":
return {
- formatTableName: defaultFormatter.formatTableName,
+ formatTablePath: defaultFormatter.formatTablePath,
formatSelectClause: (columnName: string, tableName: string) =>
`SELECT TOP 100 ${columnName} FROM ${tableName}`,
};
@@ -57,12 +58,11 @@ function getFormatter(dialect: string): SqlCodeFormatter {
case "postgres":
case "postgresql":
case "duckdb":
+ case "dremio":
// Quote column and table names to avoid raising errors on weird characters
return {
- formatTableName: (tableName: string) => {
- const parts = tableName.split(".");
- return parts.map((part) => `"${part}"`).join(".");
- },
+ formatTablePath: (tablePath: string[]) =>
+ tablePath.map((part) => `"${part}"`).join("."),
formatSelectClause: (columnName: string, tableName: string) =>
`SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,
};
@@ -114,26 +114,27 @@ export function sqlCode({
database,
dialect,
} = sqlTableContext;
- let tableName = table.name;
+ const tablePath = [table.name];
// Set the fully qualified table name based on schema and database
if (isSchemaless(schema)) {
- tableName =
- database === defaultDatabase ? tableName : `${database}.${tableName}`;
+ if (database !== defaultDatabase) {
+ tablePath.unshift(database);
+ }
} else {
// Include schema if it's not the default schema
if (schema !== defaultSchema) {
- tableName = `${schema}.${tableName}`;
+ tablePath.unshift(schema);
}
// Include database if it's not the default database
if (database !== defaultDatabase) {
- tableName = `${database}.${tableName}`;
+ tablePath.unshift(database);
}
}
const formatter = getFormatter(dialect);
- const formattedTableName = formatter.formatTableName(tableName);
+ const formattedTableName = formatter.formatTablePath(tablePath);
const selectClause = formatter.formatSelectClause(
columnName,
formattedTableName,
diff --git a/frontend/src/core/codemirror/format.ts b/frontend/src/core/codemirror/format.ts
index 800ecf21a3..297b02f267 100644
--- a/frontend/src/core/codemirror/format.ts
+++ b/frontend/src/core/codemirror/format.ts
@@ -205,6 +205,7 @@ async function getSqlFormatterDialect(
case "mongodb":
case "timescaledb":
case "datafusion":
+ case "dremio":
return sql;
case "databricks":
return spark;
diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.ts
index 435f1173f4..3a35405d2e 100644
--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts
@@ -410,6 +410,7 @@ function connectionNameToParserDialect(
case "spark":
case "databricks":
case "datafusion":
+ case "dremio":
Logger.debug("Unsupported dialect", { dialect });
return null;
default:
diff --git a/frontend/src/core/codemirror/language/languages/sql/utils.ts b/frontend/src/core/codemirror/language/languages/sql/utils.ts
index 19763a402d..2189936cf2 100644
--- a/frontend/src/core/codemirror/language/languages/sql/utils.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/utils.ts
@@ -52,6 +52,7 @@ const KNOWN_DIALECTS_ARRAY = [
"databricks",
"datafusion",
"microsoft sql server",
+ "dremio",
] as const;
const KNOWN_DIALECTS: ReadonlySet<string> = new Set(KNOWN_DIALECTS_ARRAY);
type KnownDialect = (typeof KNOWN_DIALECTS_ARRAY)[number];
@@ -115,6 +116,7 @@ export function guessDialect(
case "spark":
case "databricks":
case "datafusion":
+ case "dremio":
Logger.debug("Unsupported dialect", { dialect });
return ModifiedStandardSQL;
default:
diff --git a/marimo/_sql/engines/adbc.py b/marimo/_sql/engines/adbc.py
index c7c7427a9a..37321bed18 100644
--- a/marimo/_sql/engines/adbc.py
+++ b/marimo/_sql/engines/adbc.py
@@ -30,6 +30,22 @@ AdbcGetObjectsDepth = Literal[
"all", "catalogs", "db_schemas", "tables", "columns"
]
+ARROW_INTEGER_TYPES = frozenset(
+ f"{prefix}{bits}"
+ for prefix in ("int", "uint")
+ for bits in ("8", "16", "32", "64")
+)
+ARROW_STRING_TYPES = frozenset(
+ (
+ "binary",
+ "binary_view",
+ "large_binary",
+ "large_string",
+ "string",
+ "string_view",
+ )
+)
+
class AdbcDbApiCursor(Protocol):
description: Any
@@ -108,10 +124,22 @@ def _adbc_info_to_dialect(*, info: dict[str | int, Any]) -> str:
def _schema_field_to_data_type(external_type: str) -> DataType:
"""Map an Arrow-like dtype string to marimo DataType."""
- t = external_type.lower()
- if "bool" in t:
+ t = external_type.strip().lower()
+ if t.startswith(
+ (
+ "array<",
+ "fixed_size_list<",
+ "large_list<",
+ "list<",
+ "map<",
+ "struct<",
+ "union<",
+ )
+ ) or t.endswith("[]"):
+ return "unknown"
+ if t == "bool":
return "boolean"
- if "int" in t or "uint" in t:
+ if t in ARROW_INTEGER_TYPES:
return "integer"
if "float" in t or "double" in t or "decimal" in t:
return "number"
@@ -121,7 +149,10 @@ def _schema_field_to_data_type(external_type: str) -> DataType:
return "date"
if t.startswith("time") or " time" in t:
return "time"
- return "string"
+ # Arrow renders fixed-size binary with its width, e.g. fixed_size_binary[16].
+ if t in ARROW_STRING_TYPES or t.startswith("fixed_size_binary["):
+ return "string"
+ return "unknown"
class AdbcConnectionCatalog:
diff --git a/tests/_sql/test_adbc.py b/tests/_sql/test_adbc.py
index aa307fa7eb..1b2bb191ce 100644
--- a/tests/_sql/test_adbc.py
+++ b/tests/_sql/test_adbc.py
@@ -11,6 +11,7 @@ from marimo._sql.engines.adbc import (
AdbcConnectionCatalog,
AdbcDBAPIEngine,
_adbc_info_to_dialect,
+ _schema_field_to_data_type,
)
from marimo._sql.get_engines import get_engines_from_variables
from marimo._types.ids import VariableName
@@ -229,6 +230,7 @@ def test_adbc_info_to_dialect() -> None:
_adbc_info_to_dialect(info={"vendor_name": "Microsoft SQL Server"})
== "microsoft sql server"
)
+ assert _adbc_info_to_dialect(info={"vendor_name": "Dremio"}) == "dremio"
# Missing/blank/non-string vendor_name falls back to "sql".
assert _adbc_info_to_dialect(info={"vendor_name": " "}) == "sql"
@@ -237,6 +239,48 @@ def test_adbc_info_to_dialect() -> None:
assert _adbc_info_to_dialect(info={}) == "sql"
+ "external_type",
+ [
+ "list<$data$: bool not null>",
+ "list<$data$: int64 not null>",
+ "large_list<$data$: double>",
+ "fixed_size_list<item: string>[3]",
+ "array<int32>",
+ "int64[]",
+ " struct<id: int64, enabled: bool> ",
+ "struct<id: int64, enabled: bool>",
+ "map<string, int64>",
+ ],
+)
+def test_schema_field_to_data_type_returns_unknown_for_nested_types(
+ external_type: str,
+) -> None:
+ assert _schema_field_to_data_type(external_type) == "unknown"
+
+
+ ("external_type", "data_type"),
+ [
+ ("int8", "integer"),
+ ("int64", "integer"),
+ ("uint32", "integer"),
+ ("uint64", "integer"),
+ ("string", "string"),
+ ("large_string", "string"),
+ ("binary", "string"),
+ ("fixed_size_binary[16]", "string"),
+ ("interval", "unknown"),
+ ("dictionary<values=string, indices=int8, ordered=0>", "unknown"),
+ ("bool", "boolean"),
+ ],
+)
+def test_schema_field_to_data_type_handles_scalar_types(
+ external_type: str, data_type: str
+) -> None:
+ assert _schema_field_to_data_type(external_type) == data_type
+
+
def test_get_engines_from_variables_prefers_adbc_dbapi_engine() -> None:
conn = FakeAdbcDbApiConnection(
cursor=FakeAdbcDbApiCursor(description=None),