A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
support diff dialects in sql formatter (#7270)
Shahmir Varqhae69e048d3728
diff --git a/frontend/src/components/datasources/utils.ts b/frontend/src/components/datasources/utils.tsindex f655913c5b..279fd890e7 100644--- a/frontend/src/components/datasources/utils.ts+++ b/frontend/src/components/datasources/utils.ts@@ -66,11 +66,13 @@ function getFormatter(dialect: string): SqlCodeFormatter {`SELECT ${columnName === "*" ? "*" : `"${columnName}"`} FROM ${tableName} LIMIT 100`,};case "db2":+ case "db2i":case "mysql":case "sqlite":case "mariadb":case "cassandra":case "noql":+ case "awsathena":case "athena":case "hive":case "redshift":@@ -79,6 +81,11 @@ function getFormatter(dialect: string): SqlCodeFormatter {case "mongodb":case "oracle":case "oracledb":+ case "couchbase":+ case "tidb":+ case "spark":+ case "trino":+ case "singlestoredb":return defaultFormatter;default:logNever(dialect);diff --git a/frontend/src/core/codemirror/__tests__/format.test.ts b/frontend/src/core/codemirror/__tests__/format.test.tsindex 2648683c57..d9987693d8 100644--- a/frontend/src/core/codemirror/__tests__/format.test.ts+++ b/frontend/src/core/codemirror/__tests__/format.test.ts@@ -11,6 +11,7 @@ import { getNotebook } from "@/core/cells/cells";import type { CellId } from "@/core/cells/ids";import { notebookCellEditorViews } from "@/core/cells/utils";import { getResolvedMarimoConfig } from "@/core/config/config";+import type { ConnectionName } from "@/core/datasets/engines";import { OverridingHotkeyProvider } from "@/core/hotkeys/hotkeys";import { requestClientAtom } from "@/core/network/requests";import type { MarimoConfig } from "@/core/network/types";@@ -198,7 +199,7 @@ describe("format", () => {const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);switchLanguage(editor, { language: "sql" });- await formatSQL(editor);+ await formatSQL(editor, "duckdb" as ConnectionName);// Check that the SQL was formattedexpect(editor.state.doc.toString()).toMatchInlineSnapshot(`@@ -221,7 +222,7 @@ describe("format", () => {const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);switchLanguage(editor, { language: "python" });- await formatSQL(editor);+ await formatSQL(editor, "duckdb" as ConnectionName);// Check that the SQL was not formattedexpect(editor.state.doc.toString()).toBe(@@ -230,4 +231,29 @@ describe("format", () => {expect(updateCellCode).not.toHaveBeenCalled();});});++ it("should format SQL code with different dialect", async () => {+ const cellId = "1" as CellId;+ const editor = createEditor(+ "SELECT * FROM `table.dot` WHERE id = 1",+ cellId,+ );+ switchLanguage(editor, { language: "sql" });++ await formatSQL(editor, "mysql" as ConnectionName); // mysql uses backticks for identifiers++ expect(editor.state.doc.toString()).toMatchInlineSnapshot(`+ "SELECT+ *+ FROM+ \`table.dot\`+ WHERE+ id = 1"+ `);+ expect(updateCellCode).toHaveBeenCalledWith({+ cellId: cellId,+ code: editor.state.doc.toString(),+ formattingChange: true,+ });+ });});diff --git a/frontend/src/core/codemirror/format.ts b/frontend/src/core/codemirror/format.tsindex 661ccf3f1b..22735dc570 100644--- a/frontend/src/core/codemirror/format.ts+++ b/frontend/src/core/codemirror/format.ts@@ -3,16 +3,21 @@import { getIndentUnit } from "@codemirror/language";import { StateEffect } from "@codemirror/state";import type { EditorView } from "@codemirror/view";+import type { DialectOptions } from "sql-formatter";+import { logNever } from "@/utils/assertNever";import { Logger } from "@/utils/Logger";import { Objects } from "../../utils/objects";import { getNotebook } from "../cells/cells";import type { CellId } from "../cells/ids";import { notebookCellEditorViews } from "../cells/utils";import { getResolvedMarimoConfig } from "../config/config";+import type { ConnectionName } from "../datasets/engines";import { getRequestClient } from "../network/requests";import { cellActionsState } from "./cells/state";import { cellIdState } from "./config/extension";import { languageAdapterState } from "./language/extension";+import { SCHEMA_CACHE } from "./language/languages/sql/completion-store";+import { isKnownDialect } from "./language/languages/sql/utils";import {getEditorCodeAsPython,updateEditorCodeFromPython,@@ -77,9 +82,12 @@ export function formatAll() {* SQL formatting is much more opinionated than Python formatting, and we* don't want to tie the two together (just yet).*/-export async function formatSQL(editor: EditorView) {+export async function formatSQL(editor: EditorView, engine: ConnectionName) {// Lazy import sql-formatter- const { formatDialect, duckdb } = await import("sql-formatter");+ const { formatDialect } = await import("sql-formatter");++ const sqlDialect = SCHEMA_CACHE.getInternalDialect(engine);+ const formatterDialect = await getSqlFormatterDialect(sqlDialect);// Get language adapterconst languageAdapter = editor.state.field(languageAdapterState);@@ -90,11 +98,17 @@ export async function formatSQL(editor: EditorView) {}const codeAsSQL = editor.state.doc.toString();- const formattedSQL = formatDialect(codeAsSQL, {- dialect: duckdb,- tabWidth: tabWidth,- useTabs: false,- });+ let formattedSQL: string;+ try {+ formattedSQL = formatDialect(codeAsSQL, {+ dialect: formatterDialect,+ tabWidth: tabWidth,+ useTabs: false,+ });+ } catch (error) {+ Logger.error("Error formatting SQL", { error });+ return;+ }// Update Python in the notebook stateconst codeAsPython = languageAdapter.transformIn(formattedSQL)[0];@@ -111,3 +125,87 @@ export async function formatSQL(editor: EditorView) {effects: [formattingChangeEffect.of(true)],});}++async function getSqlFormatterDialect(+ sqlDialect: string | null,+): Promise<DialectOptions> {+ const {+ bigquery,+ db2,+ db2i,+ duckdb,+ hive,+ mariadb,+ mysql,+ tidb,+ n1ql,+ plsql,+ postgresql,+ redshift,+ spark,+ sqlite,+ sql,+ trino,+ transactsql,+ singlestoredb,+ snowflake,+ } = await import("sql-formatter");++ const defaultDialect = sql;++ if (!sqlDialect || !isKnownDialect(sqlDialect)) {+ return defaultDialect;+ }+ switch (sqlDialect) {+ case "mysql":+ return mysql;+ case "mariadb":+ return mariadb;+ case "postgres":+ case "postgresql":+ return postgresql;+ case "sqlite":+ return sqlite;+ case "db2":+ return db2;+ case "db2i":+ return db2i;+ case "hive":+ return hive;+ case "redshift":+ return redshift;+ case "snowflake":+ return snowflake;+ case "trino":+ return trino;+ case "tidb":+ return tidb;+ case "oracle":+ case "oracledb":+ return plsql;+ case "spark":+ return spark;+ case "singlestoredb":+ return singlestoredb;+ case "couchbase":+ return n1ql;+ case "bigquery":+ return bigquery;+ case "duckdb":+ return duckdb;+ case "mssql":+ case "sqlserver":+ return transactsql;+ case "athena":+ case "awsathena":+ case "cassandra":+ case "noql":+ case "flink":+ case "mongodb":+ case "timescaledb":+ return sql;+ default:+ logNever(sqlDialect);+ return defaultDialect;+ }+}diff --git a/frontend/src/core/codemirror/language/languages/sql/completion-store.ts b/frontend/src/core/codemirror/language/languages/sql/completion-store.tsindex b0a81122be..55b8194b0a 100644--- a/frontend/src/core/codemirror/language/languages/sql/completion-store.ts+++ b/frontend/src/core/codemirror/language/languages/sql/completion-store.ts@@ -122,7 +122,7 @@ class SQLCompletionStore {if (!connection) {return null;}- return connection.dialect;+ return connection.dialect.toLowerCase();}/**diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.tsindex e4a14c0685..a479d061c2 100644--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts@@ -137,7 +137,7 @@ export class SQLLanguageAdapter_completionConfig: CompletionConfig,_hotkeys: HotkeyProvider,_placeholderType: PlaceholderType,- lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },+ _lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },): Extension[] {const extensions = [// This can be updated with a dispatch effect@@ -366,6 +366,7 @@ function connectionNameToParserDialect(case "postgres":return "PostgreSQL";case "db2":+ case "db2i":return "DB2";case "mysql":return "MySQL";@@ -381,6 +382,7 @@ function connectionNameToParserDialect(case "cassandra":return "Noql";case "athena":+ case "awsathena":return "Athena";case "bigquery":return "BigQuery";@@ -398,6 +400,11 @@ function connectionNameToParserDialect(case "oracle":case "oracledb":case "timescaledb":+ case "couchbase":+ case "trino":+ case "tidb":+ case "singlestoredb":+ case "spark":Logger.debug("Unsupported dialect", { dialect });return null;default:diff --git a/frontend/src/core/codemirror/language/languages/sql/utils.ts b/frontend/src/core/codemirror/language/languages/sql/utils.tsindex 5ebf3ff40e..24e3958ec2 100644--- a/frontend/src/core/codemirror/language/languages/sql/utils.ts+++ b/frontend/src/core/codemirror/language/languages/sql/utils.ts@@ -23,7 +23,10 @@ import { Logger } from "@/utils/Logger";const KNOWN_DIALECTS_ARRAY = ["postgresql","postgres",+ "couchbase","db2",+ "db2i",+ "tidb","mysql","sqlite","mssql",@@ -32,6 +35,8 @@ const KNOWN_DIALECTS_ARRAY = ["mariadb","cassandra","noql",+ "spark",+ "awsathena","athena","bigquery","hive",@@ -39,8 +44,10 @@ const KNOWN_DIALECTS_ARRAY = ["snowflake","flink","mongodb",+ "trino","oracle","oracledb",+ "singlestoredb","timescaledb",] as const;const KNOWN_DIALECTS: ReadonlySet<string> = new Set(KNOWN_DIALECTS_ARRAY);@@ -59,6 +66,7 @@ export function guessDialect(): SQLDialect {const dialect = connection.dialect;if (!isKnownDialect(dialect)) {+ Logger.debug("Unknown dialect", { dialect });return ModifiedStandardSQL;}@@ -86,7 +94,9 @@ export function guessDialect(return BigQueryDialect;case "timescaledb":return PostgreSQL; // TimescaleDB is a PostgreSQL dialect+ case "awsathena":case "athena":+ case "db2i":case "db2":case "hive":case "redshift":@@ -94,6 +104,11 @@ export function guessDialect(case "flink":case "mongodb":case "noql":+ case "couchbase":+ case "trino":+ case "tidb":+ case "singlestoredb":+ case "spark":Logger.debug("Unsupported dialect", { dialect });return ModifiedStandardSQL;default:diff --git a/frontend/src/core/codemirror/language/panel/panel.tsx b/frontend/src/core/codemirror/language/panel/panel.tsxindex 919deebe11..d36eba03df 100644--- a/frontend/src/core/codemirror/language/panel/panel.tsx+++ b/frontend/src/core/codemirror/language/panel/panel.tsx@@ -101,7 +101,7 @@ export const LanguagePanelComponent: React.FC<{variant="text"size="icon"onClick={async () => {- await formatSQL(view);+ await formatSQL(view, metadata.engine);}}><PaintRollerIcon className="h-3 w-3" />