The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1013: Correct identifier casing when Metabot fixes SQL (#79674)
Andrei Simionescu1f807106abb4
diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.pyindex 1f799f6f4d7..5b337fddba1 100644--- a/resources/python-sources/sql_tools.py+++ b/resources/python-sources/sql_tools.py@@ -14,6 +14,7 @@ import sqlglot.optimizer.qualify as qualifyfrom sqlglot import expfrom sqlglot.dialects.clickhouse import ClickHousefrom sqlglot.errors import OptimizeError, ParseError+from sqlglot.optimizer.normalize_identifiers import normalize_identifiers# sqlglot (as of 28.6.0) renders every placeholder in ClickHouse's named-parameter# syntax `{name: Type}`, so a positional JDBC placeholder `?` (a nameless Placeholder)@@ -1648,27 +1649,90 @@ def has_metabase_templates(sql: str) -> bool:"""return bool(_METABASE_TEMPLATE_RE.search(sql))-# Dialects that require identifier quoting due to case sensitivity.-# These dialects fold unquoted identifiers to uppercase or lowercase,-# which can cause issues when the LLM generates mixed-case identifiers.-CASE_SENSITIVE_DIALECTS: set[str] = {- "snowflake", # Folds unquoted to UPPERCASE- "oracle", # Folds unquoted to UPPERCASE- "redshift", # PostgreSQL-based, folds to lowercase- "postgres", # Folds unquoted to lowercase+# Reserved words that cannot appear as bare identifiers in the target dialect, for the+# dialects whose sqlglot generator ships an empty RESERVED_KEYWORDS set. Sourced from the+# vendors' reserved-word documentation. Dialects absent here fall back to sqlglot's own+# per-dialect generator set; where that is empty too (e.g. SQLite, whose double-quote+# quirk turns an unresolved quoted identifier into a string literal), identifiers pass+# through untouched and the warehouse reports its own error.+_DIALECT_RESERVED = {+ 'postgres': frozenset("""+ all analyse analyze and any array as asc asymmetric authorization binary both case+ cast check collate collation column concurrently constraint create cross+ current_catalog current_date current_role current_schema current_time+ current_timestamp current_user default deferrable desc distinct do else end except+ false fetch for foreign freeze from full grant group having ilike in initially+ inner intersect into is isnull join lateral leading left like limit localtime+ localtimestamp natural not notnull null offset on only or order outer overlaps+ placing primary references returning right select session_user similar some+ symmetric system_user table tablesample then to trailing true union unique user+ using variadic verbose when where window with+ """.split()),+ 'snowflake': frozenset("""+ account all alter and any as between by case cast check column connect connection+ constraint create cross current current_date current_time current_timestamp+ current_user database delete distinct drop else exists false following for from+ full grant group gscluster having ilike in increment inner insert intersect into+ is issue join lateral left like localtime localtimestamp minus natural not null+ of on or order organization qualify regexp revoke right rlike row rows sample+ schema select set some start table tablesample then to trigger true try_cast+ union unique update using values view when whenever where with+ """.split()),+ 'oracle': frozenset("""+ access add all alter and any as asc audit between by char check cluster column+ column_value comment compress connect create current date decimal default delete+ desc distinct drop else exclusive exists file float for from grant group having+ identified immediate in increment index initial insert integer intersect into is+ level like lock long maxextents minus mlslabel mode modify nested_table_id noaudit+ nocompress not nowait null number of offline on online option or order pctfree+ prior public raw rename resource revoke row rowid rownum rows select session set+ share size smallint start successful synonym sysdate table then to trigger uid+ union unique update user validate values varchar varchar2 view whenever where with+ """.split()),}+def _quote_reserved_identifiers(expression, dialect):+ """Quote identifiers that collide with the target dialect's reserved words.++ The identifier is folded first, the way the server would have folded the bare name,+ so the quoting never changes what it resolves to; it only stops the server from+ reading it as a keyword. Only words reserved in the specific target dialect are+ touched: a generic list would misfire, e.g. quoting `user` on SQLite, where USER is+ not a keyword and the quoted form of a nonexistent column silently becomes a string+ literal instead of an error.+ """+ # `dialect` may carry options (`postgres, normalization_strategy=lowercase`), so key off the class.+ reserved = (dialect.generator_class.RESERVED_KEYWORDS+ | _DIALECT_RESERVED.get(type(dialect).__name__.lower(), frozenset()))+ if reserved:+ for ident in expression.find_all(exp.Identifier):+ if not ident.quoted and ident.name.lower() in reserved:+ normalize_identifiers(ident, dialect=dialect)+ ident.set('quoted', True)+ return expression+def transpile_sql(sql: str, from_dialect: str = None, to_dialect: str = None):"""Transpile sql string from one dialect to another.+ Identifiers keep the quoting they were written with: an unquoted identifier stays+ unquoted, so the database folds it exactly as it would have folded the input.+ Quoting everything (identify=True) would instead freeze the written casing, which+ breaks on dialects that fold unquoted identifiers (Snowflake uppercases, Postgres+ lowercases). The tradeoff runs the other way for objects created with quoted+ mixed-case names (ORMs emit CREATE TABLE "Orders"): those only resolve when the+ model also writes them quoted, since the bare name folds away from the stored one.+ The one exception to pass-through is an identifier that collides with the target+ dialect's reserved words: it is quoted, folded first so the quoting cannot change+ what it resolves to.+Args:sql: SQL query stringfrom_dialect: source sql dialectto_dialect: target sql dialectReturns:- JSON string with keys transpiled and use_identify on success.- On failure the object contains keys is_valid and error_message.+ JSON string with keys transpiled_sql and status on success.+ On failure the object contains keys status and error_message."""result = {}if has_metabase_templates(sql):@@ -1681,22 +1745,15 @@ def transpile_sql(sql: str, from_dialect: str = None, to_dialect: str = None):result['reason'] = 'missing_dialect'else:try:- use_identify = (from_dialect in CASE_SENSITIVE_DIALECTS- or to_dialect in CASE_SENSITIVE_DIALECTS)-- transpiled = sqlglot.transpile(- sql,- read=from_dialect,- write=to_dialect,- pretty=True,- identify=use_identify,- )-- if len(transpiled) > 1:+ write = sqlglot.Dialect.get_or_raise(to_dialect)+ expressions = sqlglot.parse(sql, read=from_dialect)++ if len(expressions) > 1:result['status'] = 'error'result['error_message'] = 'Multiple SQL statements are not supported. Please provide a single query.'else:- result['transpiled_sql'] = transpiled[0]+ ast = _quote_reserved_identifiers(expressions[0], write)+ result['transpiled_sql'] = write.generate(ast, copy=False, pretty=True)result['status'] = 'success'except Exception as e:result['status'] = 'error'diff --git a/src/metabase/metabot/tools/sql/validation.clj b/src/metabase/metabot/tools/sql/validation.cljindex aa2f7ed06f0..0e1788125f5 100644--- a/src/metabase/metabot/tools/sql/validation.clj+++ b/src/metabase/metabot/tools/sql/validation.clj@@ -38,23 +38,13 @@"sqlserver" "tsql",;; Embedded/lightweight"sqlite" "sqlite",- ;; H2 uses PostgreSQL compatibility mode in sqlglot but this causes incorrect identifier- ;; quoting (H2 folds to UPPERCASE internally, postgres quoting forces lowercase), so we- ;; skip validation. Vertica is not supported by sqlglot.+ ;; sqlglot has no H2 dialect, and its nearest stand-in, postgres mode, folds the other+ ;; way (H2 folds unquoted identifiers to uppercase, postgres to lowercase), so anything+ ;; the transpiler folds or quotes comes out wrong for H2; skip validation. Vertica is+ ;; not supported by sqlglot."h2" nil"vertica" nil})-;; Dialects where the Python transpilation layer (`sql_tools.py`) passes `identify=True` to-;; sqlglot, quoting all identifiers. These dialects fold unquoted identifiers to a specific case-;; and need quoting to preserve the LLM's intended casing. This set is defined in Python's-;; `CASE_SENSITIVE_DIALECTS`; it is documented here so that any future pure-JVM replacement-;; knows to replicate the behavior.-;;-;; "snowflake" — folds unquoted to UPPERCASE-;; "oracle" — folds unquoted to UPPERCASE-;; "redshift" — PostgreSQL-based, folds to lowercase-;; "postgres" — folds unquoted to lowercase-(defn database-id->dialect"Get dialect for database id."[db-id]diff --git a/test/metabase/metabot/tools/sql/validation_test.clj b/test/metabase/metabot/tools/sql/validation_test.cljindex 0be71f087d7..0c826438a55 100644--- a/test/metabase/metabot/tools/sql/validation_test.clj+++ b/test/metabase/metabot/tools/sql/validation_test.clj@@ -2,7 +2,8 @@(:require[clojure.string :as str][clojure.test :refer [deftest testing is]]- [metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation]))+ [metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation]+ [metabase.sql-parsing.core :as sql-parsing]));;;; contains-template-tags?@@ -84,14 +85,14 @@(def ^:private transpilation-cases"Testing _context_ describe a test case. Apart from that, pretty formatting is checked by comparisons of raw output- stirngs in `:transpiled-sql`."- [{:context "Snowflake should quote identifiers in normalized SQL"+ strings in `:transpiled-sql`."+ [{:context "Snowflake identifiers keep the quoting they were written with":dialect "snowflake" :sql "SELECT id FROM PUBLIC.users"- :expected {:valid? true :transpiled-sql "SELECT\n \"id\"\nFROM \"PUBLIC\".\"users\""}}- {:context "PostgreSQL should quote identifiers in normalized SQL"+ :expected {:valid? true :transpiled-sql "SELECT\n id\nFROM PUBLIC.users"}}+ {:context "PostgreSQL identifiers keep the quoting they were written with":dialect "postgres" :sql "SELECT id FROM public.users"- :expected {:valid? true :transpiled-sql "SELECT\n \"id\"\nFROM \"public\".\"users\""}}- {:context "MySQL should not add identifier quoting (not case-sensitive)"+ :expected {:valid? true :transpiled-sql "SELECT\n id\nFROM public.users"}}+ {:context "MySQL identifiers keep the quoting they were written with":dialect "mysql" :sql "SELECT id FROM users":expected {:valid? true :transpiled-sql "SELECT\n id\nFROM users"}}{:context "Multiple SQL statements should be rejected"@@ -101,10 +102,81 @@{:context "Transpilation should preserve the query's logical structure":dialect "snowflake" :sql "SELECT a, b FROM t WHERE x > 1 ORDER BY a":expected {:valid? true :transpiled-sql- "SELECT\n \"a\",\n \"b\"\nFROM \"t\"\nWHERE\n \"x\" > 1\nORDER BY\n \"a\""}}])+ "SELECT\n a,\n b\nFROM t\nWHERE\n x > 1\nORDER BY\n a"}}])(deftest transpile-sql-test(doseq [{:keys [context dialect expected sql]} transpilation-cases](testing context(is (=? expected(metabot.tools.sql.validation/validate-sql dialect sql))))))++;;;; identifier quoting preservation+;;+;; Transpilation must not add identifier quoting: dialects like Snowflake and Postgres fold+;; unquoted identifiers, so quoting locks in the written casing and breaks references that the+;; database itself would have resolved (BOT-1013). Identifiers keep the quoting they were+;; written with, in both fold directions. The one exception is a name that collides with the+;; target dialect's reserved words, which is quoted in the dialect's folded case so it still+;; resolves to the same object.++(defn- transpiled-sql [dialect sql]+ (:transpiled-sql (metabot.tools.sql.validation/validate-sql dialect sql)))++(def ^:private quoting-cases+ [{:context "unquoted identifiers stay unquoted through an uppercase-folding dialect"+ :dialect "snowflake" :sql "select SubTotal, \"Weird Col\" from Orders as O where o.Total > 0"+ :expected "SELECT\n SubTotal,\n \"Weird Col\"\nFROM Orders AS O\nWHERE\n o.Total > 0"}+ {:context "unquoted identifiers stay unquoted through a lowercase-folding dialect"+ :dialect "postgres" :sql "SELECT subtotal AS \"Total\" FROM orders WHERE total > 0"+ :expected "SELECT\n subtotal AS \"Total\"\nFROM orders\nWHERE\n total > 0"}+ {:context "CTE declarations and references keep their spelling"+ :dialect "postgres" :sql "WITH Orders AS (SELECT 1 AS id) SELECT id FROM ORDERS"+ :expected "WITH Orders AS (\n SELECT\n 1 AS id\n)\nSELECT\n id\nFROM ORDERS"}+ {:context "table-valued functions transpile"+ :dialect "postgres" :sql "SELECT * FROM generate_series(1, 10) AS g(n)"+ :expected "SELECT\n *\nFROM GENERATE_SERIES(1, 10) AS g(n)"}+ {:context "reserved words are quoted in the dialect's folded case, lowercase dialect"+ :dialect "postgres" :sql "select Order, user from t"+ :expected "SELECT\n \"order\",\n \"user\"\nFROM t"}+ {:context "reserved words are quoted in the dialect's folded case, uppercase dialect"+ :dialect "snowflake" :sql "select order from t"+ :expected "SELECT\n \"ORDER\"\nFROM t"}+ {:context "Snowflake-specific reserved words are covered, not just the ANSI ones"+ :dialect "snowflake" :sql "SELECT BY FROM t"+ :expected "SELECT\n \"BY\"\nFROM t"}+ {:context "Oracle-specific reserved words are covered, not just the ANSI ones"+ :dialect "oracle" :sql "SELECT DATE, LEVEL, USER FROM t"+ :expected "SELECT\n \"DATE\",\n \"LEVEL\",\n \"USER\"\nFROM t"}+ {:context "a word another dialect reserves stays bare where the target does not reserve it:+ on SQLite a quoted unresolved identifier silently becomes a string literal"+ :dialect "sqlite" :sql "SELECT user FROM orders LIMIT 3"+ :expected "SELECT\n user\nFROM orders\nLIMIT 3"}])++(deftest ^:parallel quoting-preservation-test+ (doseq [{:keys [context dialect expected sql]} quoting-cases]+ (testing context+ (is (= expected (transpiled-sql dialect sql))))))++(deftest ^:parallel transpiled-sql-resolves-test+ (testing "transpiled SQL must resolve against the folded-case names the warehouse reports;+ quoting it (the old identify=True behavior) makes exactly these fail"+ (doseq [[dialect sql default-schema sqlglot-schema]+ [["snowflake" "select subtotal from orders"+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT"}}}]+ ["snowflake" "select SubTotal from Orders as O where o.Total > 0"+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT", "TOTAL" "FLOAT"}}}]+ ["postgres" "SELECT subtotal AS \"Total\" FROM orders WHERE total > 0"+ "public" {"public" {"orders" {"subtotal" "FLOAT", "total" "FLOAT"}}}]+ ["postgres" "SELECT SubTotal FROM Orders WHERE Total > 0"+ "public" {"public" {"orders" {"subtotal" "FLOAT", "total" "FLOAT"}}}]+ ["postgres" "SELECT \"order\" FROM t"+ "public" {"public" {"t" {"order" "FLOAT"}}}]+ ["postgres" "SELECT order FROM t"+ "public" {"public" {"t" {"order" "FLOAT"}}}]+ ["postgres" "WITH Orders AS (SELECT 1 AS id) SELECT id FROM ORDERS"+ "public" {"public" {"unrelated" {"x" "INT"}}}]+ ["snowflake" "WITH x AS (SELECT * FROM orders) SELECT subtotal FROM x"+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT"}}}]]]+ (testing sql+ (is (= "ok" (:status (sql-parsing/validate-query+ dialect (transpiled-sql dialect sql) default-schema sqlglot-schema))))))))diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.cljindex 19393a99457..e09473f9452 100644--- a/test/metabase/sql_tools/core_test.clj+++ b/test/metabase/sql_tools/core_test.clj@@ -117,34 +117,16 @@;; transpile-sql is only implemented for the :sqlglot backend, so these tests bind it directly;; rather than using test-parser-backends.-(deftest ^:parallel transpile-sql-snowflake-quotes-identifiers-test+(deftest ^:parallel transpile-sql-no-added-quoting-test(binding [sql-tools.settings/*parser-backend-override* :sqlglot]- (testing "Snowflake should quote identifiers (case-sensitive dialect)"- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM PUBLIC.users"- "snowflake" "snowflake")]- (is (= :success status))- (is (some? transpiled-sql))- (is (str/includes? transpiled-sql "\"PUBLIC\""))- (is (str/includes? transpiled-sql "\"users\""))))))--(deftest ^:parallel transpile-sql-postgres-quotes-identifiers-test- (binding [sql-tools.settings/*parser-backend-override* :sqlglot]- (testing "PostgreSQL should quote identifiers (case-sensitive dialect)"- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM public.users"- "postgres" "postgres")]- (is (= :success status))- (is (some? transpiled-sql))- (is (str/includes? transpiled-sql "\"public\""))- (is (str/includes? transpiled-sql "\"users\""))))))--(deftest ^:parallel transpile-sql-mysql-no-quoting-test- (binding [sql-tools.settings/*parser-backend-override* :sqlglot]- (testing "MySQL should not add double-quote identifier quoting (not case-sensitive)"- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM users"- "mysql" "mysql")]- (is (= :success status))- (is (some? transpiled-sql))- (is (not (str/includes? transpiled-sql "\"")))))))+ (testing "unquoted identifiers stay unquoted, so folding dialects resolve them as written"+ (doseq [dialect ["snowflake" "postgres" "mysql"]]+ (testing dialect+ (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM public.users"+ dialect dialect)]+ (is (= :success status))+ (is (some? transpiled-sql))+ (is (not (re-find #"[\"`]" transpiled-sql)))))))))(deftest ^:parallel transpile-sql-multi-statement-rejected-test(binding [sql-tools.settings/*parser-backend-override* :sqlglot]