Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

support diff dialects in sql formatter (#7270)

Shahmir Varqhae69e048d3728
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.ts
index f655913c5b..279fd890e7 100644
--- a/frontend/src/components/datasources/utils.ts
+++ b/frontend/src/components/datasources/utils.ts
@@ -66,11 +66,13 @@ function getFormatter(dialect: string): SqlCodeFormatter {
`SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,
};
case "db2":
+ case "db2i":
case "mysql":
case "sqlite":
case "mariadb":
case "cassandra":
case "noql":
+ case "awsathena":
case "athena":
case "hive":
case "redshift":
@@ -79,6 +81,11 @@ function getFormatter(dialect: string): SqlCodeFormatter {
case "mongodb":
case "oracle":
case "oracledb":
+ case "couchbase":
+ case "tidb":
+ case "spark":
+ case "trino":
+ case "singlestoredb":
return defaultFormatter;
default:
logNever(dialect);
diff --git a/frontend/src/core/codemirror/__tests__/format.test.ts b/frontend/src/core/codemirror/__tests__/format.test.ts
index 2648683c57..d9987693d8 100644
--- a/frontend/src/core/codemirror/__tests__/format.test.ts
+++ b/frontend/src/core/codemirror/__tests__/format.test.ts
@@ -11,6 +11,7 @@ import { getNotebook } from "@/core/cells/cells";
import type { CellId } from "@/core/cells/ids";
import { notebookCellEditorViews } from "@/core/cells/utils";
import { getResolvedMarimoConfig } from "@/core/config/config";
+import type { ConnectionName } from "@/core/datasets/engines";
import { OverridingHotkeyProvider } from "@/core/hotkeys/hotkeys";
import { requestClientAtom } from "@/core/network/requests";
import type { MarimoConfig } from "@/core/network/types";
@@ -198,7 +199,7 @@ describe("format", () => {
const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
switchLanguage(editor, { language: "sql" });
- await formatSQL(editor);
+ await formatSQL(editor, "duckdb" as ConnectionName);
// Check that the SQL was formatted
expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
@@ -221,7 +222,7 @@ describe("format", () => {
const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
switchLanguage(editor, { language: "python" });
- await formatSQL(editor);
+ await formatSQL(editor, "duckdb" as ConnectionName);
// Check that the SQL was not formatted
expect(editor.state.doc.toString()).toBe(
@@ -230,4 +231,29 @@ describe("format", () => {
expect(updateCellCode).not.toHaveBeenCalled();
});
});
+
+ it("should format SQL code with different dialect", async () => {
+ const cellId = "1" as CellId;
+ const editor = createEditor(
+ "SELECT * FROM `table.dot` WHERE id = 1",
+ cellId,
+ );
+ switchLanguage(editor, { language: "sql" });
+
+ await formatSQL(editor, "mysql" as ConnectionName); // mysql uses backticks for identifiers
+
+ expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
+ "SELECT
+ *
+ FROM
+ \`table.dot\`
+ WHERE
+ id = 1"
+ `);
+ expect(updateCellCode).toHaveBeenCalledWith({
+ cellId: cellId,
+ code: editor.state.doc.toString(),
+ formattingChange: true,
+ });
+ });
});
diff --git a/frontend/src/core/codemirror/format.ts b/frontend/src/core/codemirror/format.ts
index 661ccf3f1b..22735dc570 100644
--- a/frontend/src/core/codemirror/format.ts
+++ b/frontend/src/core/codemirror/format.ts
@@ -3,16 +3,21 @@
import { getIndentUnit } from "@codemirror/language";
import { StateEffect } from "@codemirror/state";
import type { EditorView } from "@codemirror/view";
+import type { DialectOptions } from "sql-formatter";
+import { logNever } from "@/utils/assertNever";
import { Logger } from "@/utils/Logger";
import { Objects } from "../../utils/objects";
import { getNotebook } from "../cells/cells";
import type { CellId } from "../cells/ids";
import { notebookCellEditorViews } from "../cells/utils";
import { getResolvedMarimoConfig } from "../config/config";
+import type { ConnectionName } from "../datasets/engines";
import { getRequestClient } from "../network/requests";
import { cellActionsState } from "./cells/state";
import { cellIdState } from "./config/extension";
import { languageAdapterState } from "./language/extension";
+import { SCHEMA_CACHE } from "./language/languages/sql/completion-store";
+import { isKnownDialect } from "./language/languages/sql/utils";
import {
getEditorCodeAsPython,
updateEditorCodeFromPython,
@@ -77,9 +82,12 @@ export function formatAll() {
* SQL formatting is much more opinionated than Python formatting, and we
* don't want to tie the two together (just yet).
*/
-export async function formatSQL(editor: EditorView) {
+export async function formatSQL(editor: EditorView, engine: ConnectionName) {
// Lazy import sql-formatter
- const { formatDialect, duckdb } = await import("sql-formatter");
+ const { formatDialect } = await import("sql-formatter");
+
+ const sqlDialect = SCHEMA_CACHE.getInternalDialect(engine);
+ const formatterDialect = await getSqlFormatterDialect(sqlDialect);
// Get language adapter
const languageAdapter = editor.state.field(languageAdapterState);
@@ -90,11 +98,17 @@ export async function formatSQL(editor: EditorView) {
}
const codeAsSQL = editor.state.doc.toString();
- const formattedSQL = formatDialect(codeAsSQL, {
- dialect: duckdb,
- tabWidth: tabWidth,
- useTabs: false,
- });
+ let formattedSQL: string;
+ try {
+ formattedSQL = formatDialect(codeAsSQL, {
+ dialect: formatterDialect,
+ tabWidth: tabWidth,
+ useTabs: false,
+ });
+ } catch (error) {
+ Logger.error("Error formatting SQL", { error });
+ return;
+ }
// Update Python in the notebook state
const codeAsPython = languageAdapter.transformIn(formattedSQL)[0];
@@ -111,3 +125,87 @@ export async function formatSQL(editor: EditorView) {
effects: [formattingChangeEffect.of(true)],
});
}
+
+async function getSqlFormatterDialect(
+ sqlDialect: string | null,
+): Promise<DialectOptions> {
+ const {
+ bigquery,
+ db2,
+ db2i,
+ duckdb,
+ hive,
+ mariadb,
+ mysql,
+ tidb,
+ n1ql,
+ plsql,
+ postgresql,
+ redshift,
+ spark,
+ sqlite,
+ sql,
+ trino,
+ transactsql,
+ singlestoredb,
+ snowflake,
+ } = await import("sql-formatter");
+
+ const defaultDialect = sql;
+
+ if (!sqlDialect || !isKnownDialect(sqlDialect)) {
+ return defaultDialect;
+ }
+ switch (sqlDialect) {
+ case "mysql":
+ return mysql;
+ case "mariadb":
+ return mariadb;
+ case "postgres":
+ case "postgresql":
+ return postgresql;
+ case "sqlite":
+ return sqlite;
+ case "db2":
+ return db2;
+ case "db2i":
+ return db2i;
+ case "hive":
+ return hive;
+ case "redshift":
+ return redshift;
+ case "snowflake":
+ return snowflake;
+ case "trino":
+ return trino;
+ case "tidb":
+ return tidb;
+ case "oracle":
+ case "oracledb":
+ return plsql;
+ case "spark":
+ return spark;
+ case "singlestoredb":
+ return singlestoredb;
+ case "couchbase":
+ return n1ql;
+ case "bigquery":
+ return bigquery;
+ case "duckdb":
+ return duckdb;
+ case "mssql":
+ case "sqlserver":
+ return transactsql;
+ case "athena":
+ case "awsathena":
+ case "cassandra":
+ case "noql":
+ case "flink":
+ case "mongodb":
+ case "timescaledb":
+ return sql;
+ default:
+ logNever(sqlDialect);
+ return defaultDialect;
+ }
+}
diff --git a/frontend/src/core/codemirror/language/languages/sql/completion-store.ts b/frontend/src/core/codemirror/language/languages/sql/completion-store.ts
index b0a81122be..55b8194b0a 100644
--- a/frontend/src/core/codemirror/language/languages/sql/completion-store.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/completion-store.ts
@@ -122,7 +122,7 @@ class SQLCompletionStore {
if (!connection) {
return null;
}
- return connection.dialect;
+ return connection.dialect.toLowerCase();
}
/**
diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.ts
index e4a14c0685..a479d061c2 100644
--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts
@@ -137,7 +137,7 @@ export class SQLLanguageAdapter
_completionConfig: CompletionConfig,
_hotkeys: HotkeyProvider,
_placeholderType: PlaceholderType,
- lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
+ _lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
): Extension[] {
const extensions = [
// This can be updated with a dispatch effect
@@ -366,6 +366,7 @@ function connectionNameToParserDialect(
case "postgres":
return "PostgreSQL";
case "db2":
+ case "db2i":
return "DB2";
case "mysql":
return "MySQL";
@@ -381,6 +382,7 @@ function connectionNameToParserDialect(
case "cassandra":
return "Noql";
case "athena":
+ case "awsathena":
return "Athena";
case "bigquery":
return "BigQuery";
@@ -398,6 +400,11 @@ function connectionNameToParserDialect(
case "oracle":
case "oracledb":
case "timescaledb":
+ case "couchbase":
+ case "trino":
+ case "tidb":
+ case "singlestoredb":
+ case "spark":
Logger.debug("Unsupported dialect", { dialect });
return null;
default:
diff --git a/frontend/src/core/codemirror/language/languages/sql/utils.ts b/frontend/src/core/codemirror/language/languages/sql/utils.ts
index 5ebf3ff40e..24e3958ec2 100644
--- a/frontend/src/core/codemirror/language/languages/sql/utils.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/utils.ts
@@ -23,7 +23,10 @@ import { Logger } from "@/utils/Logger";
const KNOWN_DIALECTS_ARRAY = [
"postgresql",
"postgres",
+ "couchbase",
"db2",
+ "db2i",
+ "tidb",
"mysql",
"sqlite",
"mssql",
@@ -32,6 +35,8 @@ const KNOWN_DIALECTS_ARRAY = [
"mariadb",
"cassandra",
"noql",
+ "spark",
+ "awsathena",
"athena",
"bigquery",
"hive",
@@ -39,8 +44,10 @@ const KNOWN_DIALECTS_ARRAY = [
"snowflake",
"flink",
"mongodb",
+ "trino",
"oracle",
"oracledb",
+ "singlestoredb",
"timescaledb",
] as const;
const KNOWN_DIALECTS: ReadonlySet<string> = new Set(KNOWN_DIALECTS_ARRAY);
@@ -59,6 +66,7 @@ export function guessDialect(
): SQLDialect {
const dialect = connection.dialect;
if (!isKnownDialect(dialect)) {
+ Logger.debug("Unknown dialect", { dialect });
return ModifiedStandardSQL;
}
@@ -86,7 +94,9 @@ export function guessDialect(
return BigQueryDialect;
case "timescaledb":
return PostgreSQL; // TimescaleDB is a PostgreSQL dialect
+ case "awsathena":
case "athena":
+ case "db2i":
case "db2":
case "hive":
case "redshift":
@@ -94,6 +104,11 @@ export function guessDialect(
case "flink":
case "mongodb":
case "noql":
+ case "couchbase":
+ case "trino":
+ case "tidb":
+ case "singlestoredb":
+ case "spark":
Logger.debug("Unsupported dialect", { dialect });
return ModifiedStandardSQL;
default:
diff --git a/frontend/src/core/codemirror/language/panel/panel.tsx b/frontend/src/core/codemirror/language/panel/panel.tsx
index 919deebe11..d36eba03df 100644
--- a/frontend/src/core/codemirror/language/panel/panel.tsx
+++ b/frontend/src/core/codemirror/language/panel/panel.tsx
@@ -101,7 +101,7 @@ export const LanguagePanelComponent: React.FC<{
variant="text"
size="icon"
onClick={async () => {
- await formatSQL(view);
+ await formatSQL(view, metadata.engine);
}}
>
<PaintRollerIcon className="h-3 w-3" />