Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1013: Correct identifier casing when Metabot fixes SQL (#79674)

Andrei Simionescu1f807106abb4
diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.py
index 1f799f6f4d7..5b337fddba1 100644
--- a/resources/python-sources/sql_tools.py
+++ b/resources/python-sources/sql_tools.py
@@ -14,6 +14,7 @@ import sqlglot.optimizer.qualify as qualify
from sqlglot import exp
from sqlglot.dialects.clickhouse import ClickHouse
from sqlglot.errors import OptimizeError, ParseError
+from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
# sqlglot (as of 28.6.0) renders every placeholder in ClickHouse's named-parameter
# syntax `{name: Type}`, so a positional JDBC placeholder `?` (a nameless Placeholder)
@@ -1648,27 +1649,90 @@ def has_metabase_templates(sql: str) -> bool:
"""
return bool(_METABASE_TEMPLATE_RE.search(sql))
-# Dialects that require identifier quoting due to case sensitivity.
-# These dialects fold unquoted identifiers to uppercase or lowercase,
-# which can cause issues when the LLM generates mixed-case identifiers.
-CASE_SENSITIVE_DIALECTS: set[str] = {
- "snowflake", # Folds unquoted to UPPERCASE
- "oracle", # Folds unquoted to UPPERCASE
- "redshift", # PostgreSQL-based, folds to lowercase
- "postgres", # Folds unquoted to lowercase
+# Reserved words that cannot appear as bare identifiers in the target dialect, for the
+# dialects whose sqlglot generator ships an empty RESERVED_KEYWORDS set. Sourced from the
+# vendors' reserved-word documentation. Dialects absent here fall back to sqlglot's own
+# per-dialect generator set; where that is empty too (e.g. SQLite, whose double-quote
+# quirk turns an unresolved quoted identifier into a string literal), identifiers pass
+# through untouched and the warehouse reports its own error.
+_DIALECT_RESERVED = {
+ 'postgres': frozenset("""
+ all analyse analyze and any array as asc asymmetric authorization binary both case
+ cast check collate collation column concurrently constraint create cross
+ current_catalog current_date current_role current_schema current_time
+ current_timestamp current_user default deferrable desc distinct do else end except
+ false fetch for foreign freeze from full grant group having ilike in initially
+ inner intersect into is isnull join lateral leading left like limit localtime
+ localtimestamp natural not notnull null offset on only or order outer overlaps
+ placing primary references returning right select session_user similar some
+ symmetric system_user table tablesample then to trailing true union unique user
+ using variadic verbose when where window with
+ """.split()),
+ 'snowflake': frozenset("""
+ account all alter and any as between by case cast check column connect connection
+ constraint create cross current current_date current_time current_timestamp
+ current_user database delete distinct drop else exists false following for from
+ full grant group gscluster having ilike in increment inner insert intersect into
+ is issue join lateral left like localtime localtimestamp minus natural not null
+ of on or order organization qualify regexp revoke right rlike row rows sample
+ schema select set some start table tablesample then to trigger true try_cast
+ union unique update using values view when whenever where with
+ """.split()),
+ 'oracle': frozenset("""
+ access add all alter and any as asc audit between by char check cluster column
+ column_value comment compress connect create current date decimal default delete
+ desc distinct drop else exclusive exists file float for from grant group having
+ identified immediate in increment index initial insert integer intersect into is
+ level like lock long maxextents minus mlslabel mode modify nested_table_id noaudit
+ nocompress not nowait null number of offline on online option or order pctfree
+ prior public raw rename resource revoke row rowid rownum rows select session set
+ share size smallint start successful synonym sysdate table then to trigger uid
+ union unique update user validate values varchar varchar2 view whenever where with
+ """.split()),
}
+def _quote_reserved_identifiers(expression, dialect):
+ """Quote identifiers that collide with the target dialect's reserved words.
+
+ The identifier is folded first, the way the server would have folded the bare name,
+ so the quoting never changes what it resolves to; it only stops the server from
+ reading it as a keyword. Only words reserved in the specific target dialect are
+ touched: a generic list would misfire, e.g. quoting `user` on SQLite, where USER is
+ not a keyword and the quoted form of a nonexistent column silently becomes a string
+ literal instead of an error.
+ """
+ # `dialect` may carry options (`postgres, normalization_strategy=lowercase`), so key off the class.
+ reserved = (dialect.generator_class.RESERVED_KEYWORDS
+ | _DIALECT_RESERVED.get(type(dialect).__name__.lower(), frozenset()))
+ if reserved:
+ for ident in expression.find_all(exp.Identifier):
+ if not ident.quoted and ident.name.lower() in reserved:
+ normalize_identifiers(ident, dialect=dialect)
+ ident.set('quoted', True)
+ return expression
+
def transpile_sql(sql: str, from_dialect: str = None, to_dialect: str = None):
"""Transpile sql string from one dialect to another.
+ Identifiers keep the quoting they were written with: an unquoted identifier stays
+ unquoted, so the database folds it exactly as it would have folded the input.
+ Quoting everything (identify=True) would instead freeze the written casing, which
+ breaks on dialects that fold unquoted identifiers (Snowflake uppercases, Postgres
+ lowercases). The tradeoff runs the other way for objects created with quoted
+ mixed-case names (ORMs emit CREATE TABLE "Orders"): those only resolve when the
+ model also writes them quoted, since the bare name folds away from the stored one.
+ The one exception to pass-through is an identifier that collides with the target
+ dialect's reserved words: it is quoted, folded first so the quoting cannot change
+ what it resolves to.
+
Args:
sql: SQL query string
from_dialect: source sql dialect
to_dialect: target sql dialect
Returns:
- JSON string with keys transpiled and use_identify on success.
- On failure the object contains keys is_valid and error_message.
+ JSON string with keys transpiled_sql and status on success.
+ On failure the object contains keys status and error_message.
"""
result = {}
if has_metabase_templates(sql):
@@ -1681,22 +1745,15 @@ def transpile_sql(sql: str, from_dialect: str = None, to_dialect: str = None):
result['reason'] = 'missing_dialect'
else:
try:
- use_identify = (from_dialect in CASE_SENSITIVE_DIALECTS
- or to_dialect in CASE_SENSITIVE_DIALECTS)
-
- transpiled = sqlglot.transpile(
- sql,
- read=from_dialect,
- write=to_dialect,
- pretty=True,
- identify=use_identify,
- )
-
- if len(transpiled) > 1:
+ write = sqlglot.Dialect.get_or_raise(to_dialect)
+ expressions = sqlglot.parse(sql, read=from_dialect)
+
+ if len(expressions) > 1:
result['status'] = 'error'
result['error_message'] = 'Multiple SQL statements are not supported. Please provide a single query.'
else:
- result['transpiled_sql'] = transpiled[0]
+ ast = _quote_reserved_identifiers(expressions[0], write)
+ result['transpiled_sql'] = write.generate(ast, copy=False, pretty=True)
result['status'] = 'success'
except Exception as e:
result['status'] = 'error'
diff --git a/src/metabase/metabot/tools/sql/validation.clj b/src/metabase/metabot/tools/sql/validation.clj
index aa2f7ed06f0..0e1788125f5 100644
--- a/src/metabase/metabot/tools/sql/validation.clj
+++ b/src/metabase/metabot/tools/sql/validation.clj
@@ -38,23 +38,13 @@
"sqlserver" "tsql",
;; Embedded/lightweight
"sqlite" "sqlite",
- ;; H2 uses PostgreSQL compatibility mode in sqlglot but this causes incorrect identifier
- ;; quoting (H2 folds to UPPERCASE internally, postgres quoting forces lowercase), so we
- ;; skip validation. Vertica is not supported by sqlglot.
+ ;; sqlglot has no H2 dialect, and its nearest stand-in, postgres mode, folds the other
+ ;; way (H2 folds unquoted identifiers to uppercase, postgres to lowercase), so anything
+ ;; the transpiler folds or quotes comes out wrong for H2; skip validation. Vertica is
+ ;; not supported by sqlglot.
"h2" nil
"vertica" nil})
-;; Dialects where the Python transpilation layer (`sql_tools.py`) passes `identify=True` to
-;; sqlglot, quoting all identifiers. These dialects fold unquoted identifiers to a specific case
-;; and need quoting to preserve the LLM's intended casing. This set is defined in Python's
-;; `CASE_SENSITIVE_DIALECTS`; it is documented here so that any future pure-JVM replacement
-;; knows to replicate the behavior.
-;;
-;; "snowflake" — folds unquoted to UPPERCASE
-;; "oracle" — folds unquoted to UPPERCASE
-;; "redshift" — PostgreSQL-based, folds to lowercase
-;; "postgres" — folds unquoted to lowercase
-
(defn database-id->dialect
"Get dialect for database id."
[db-id]
diff --git a/test/metabase/metabot/tools/sql/validation_test.clj b/test/metabase/metabot/tools/sql/validation_test.clj
index 0be71f087d7..0c826438a55 100644
--- a/test/metabase/metabot/tools/sql/validation_test.clj
+++ b/test/metabase/metabot/tools/sql/validation_test.clj
@@ -2,7 +2,8 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest testing is]]
- [metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation]))
+ [metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation]
+ [metabase.sql-parsing.core :as sql-parsing]))
;;;; contains-template-tags?
@@ -84,14 +85,14 @@
(def ^:private transpilation-cases
"Testing _context_ describe a test case. Apart from that, pretty formatting is checked by comparisons of raw output
- stirngs in `:transpiled-sql`."
- [{:context "Snowflake should quote identifiers in normalized SQL"
+ strings in `:transpiled-sql`."
+ [{:context "Snowflake identifiers keep the quoting they were written with"
:dialect "snowflake" :sql "SELECT id FROM PUBLIC.users"
- :expected {:valid? true :transpiled-sql "SELECT\n \"id\"\nFROM \"PUBLIC\".\"users\""}}
- {:context "PostgreSQL should quote identifiers in normalized SQL"
+ :expected {:valid? true :transpiled-sql "SELECT\n id\nFROM PUBLIC.users"}}
+ {:context "PostgreSQL identifiers keep the quoting they were written with"
:dialect "postgres" :sql "SELECT id FROM public.users"
- :expected {:valid? true :transpiled-sql "SELECT\n \"id\"\nFROM \"public\".\"users\""}}
- {:context "MySQL should not add identifier quoting (not case-sensitive)"
+ :expected {:valid? true :transpiled-sql "SELECT\n id\nFROM public.users"}}
+ {:context "MySQL identifiers keep the quoting they were written with"
:dialect "mysql" :sql "SELECT id FROM users"
:expected {:valid? true :transpiled-sql "SELECT\n id\nFROM users"}}
{:context "Multiple SQL statements should be rejected"
@@ -101,10 +102,81 @@
{:context "Transpilation should preserve the query's logical structure"
:dialect "snowflake" :sql "SELECT a, b FROM t WHERE x > 1 ORDER BY a"
:expected {:valid? true :transpiled-sql
- "SELECT\n \"a\",\n \"b\"\nFROM \"t\"\nWHERE\n \"x\" > 1\nORDER BY\n \"a\""}}])
+ "SELECT\n a,\n b\nFROM t\nWHERE\n x > 1\nORDER BY\n a"}}])
(deftest transpile-sql-test
(doseq [{:keys [context dialect expected sql]} transpilation-cases]
(testing context
(is (=? expected
(metabot.tools.sql.validation/validate-sql dialect sql))))))
+
+;;;; identifier quoting preservation
+;;
+;; Transpilation must not add identifier quoting: dialects like Snowflake and Postgres fold
+;; unquoted identifiers, so quoting locks in the written casing and breaks references that the
+;; database itself would have resolved (BOT-1013). Identifiers keep the quoting they were
+;; written with, in both fold directions. The one exception is a name that collides with the
+;; target dialect's reserved words, which is quoted in the dialect's folded case so it still
+;; resolves to the same object.
+
+(defn- transpiled-sql [dialect sql]
+ (:transpiled-sql (metabot.tools.sql.validation/validate-sql dialect sql)))
+
+(def ^:private quoting-cases
+ [{:context "unquoted identifiers stay unquoted through an uppercase-folding dialect"
+ :dialect "snowflake" :sql "select SubTotal, \"Weird Col\" from Orders as O where o.Total > 0"
+ :expected "SELECT\n SubTotal,\n \"Weird Col\"\nFROM Orders AS O\nWHERE\n o.Total > 0"}
+ {:context "unquoted identifiers stay unquoted through a lowercase-folding dialect"
+ :dialect "postgres" :sql "SELECT subtotal AS \"Total\" FROM orders WHERE total > 0"
+ :expected "SELECT\n subtotal AS \"Total\"\nFROM orders\nWHERE\n total > 0"}
+ {:context "CTE declarations and references keep their spelling"
+ :dialect "postgres" :sql "WITH Orders AS (SELECT 1 AS id) SELECT id FROM ORDERS"
+ :expected "WITH Orders AS (\n SELECT\n 1 AS id\n)\nSELECT\n id\nFROM ORDERS"}
+ {:context "table-valued functions transpile"
+ :dialect "postgres" :sql "SELECT * FROM generate_series(1, 10) AS g(n)"
+ :expected "SELECT\n *\nFROM GENERATE_SERIES(1, 10) AS g(n)"}
+ {:context "reserved words are quoted in the dialect's folded case, lowercase dialect"
+ :dialect "postgres" :sql "select Order, user from t"
+ :expected "SELECT\n \"order\",\n \"user\"\nFROM t"}
+ {:context "reserved words are quoted in the dialect's folded case, uppercase dialect"
+ :dialect "snowflake" :sql "select order from t"
+ :expected "SELECT\n \"ORDER\"\nFROM t"}
+ {:context "Snowflake-specific reserved words are covered, not just the ANSI ones"
+ :dialect "snowflake" :sql "SELECT BY FROM t"
+ :expected "SELECT\n \"BY\"\nFROM t"}
+ {:context "Oracle-specific reserved words are covered, not just the ANSI ones"
+ :dialect "oracle" :sql "SELECT DATE, LEVEL, USER FROM t"
+ :expected "SELECT\n \"DATE\",\n \"LEVEL\",\n \"USER\"\nFROM t"}
+ {:context "a word another dialect reserves stays bare where the target does not reserve it:
+ on SQLite a quoted unresolved identifier silently becomes a string literal"
+ :dialect "sqlite" :sql "SELECT user FROM orders LIMIT 3"
+ :expected "SELECT\n user\nFROM orders\nLIMIT 3"}])
+
+(deftest ^:parallel quoting-preservation-test
+ (doseq [{:keys [context dialect expected sql]} quoting-cases]
+ (testing context
+ (is (= expected (transpiled-sql dialect sql))))))
+
+(deftest ^:parallel transpiled-sql-resolves-test
+ (testing "transpiled SQL must resolve against the folded-case names the warehouse reports;
+ quoting it (the old identify=True behavior) makes exactly these fail"
+ (doseq [[dialect sql default-schema sqlglot-schema]
+ [["snowflake" "select subtotal from orders"
+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT"}}}]
+ ["snowflake" "select SubTotal from Orders as O where o.Total > 0"
+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT", "TOTAL" "FLOAT"}}}]
+ ["postgres" "SELECT subtotal AS \"Total\" FROM orders WHERE total > 0"
+ "public" {"public" {"orders" {"subtotal" "FLOAT", "total" "FLOAT"}}}]
+ ["postgres" "SELECT SubTotal FROM Orders WHERE Total > 0"
+ "public" {"public" {"orders" {"subtotal" "FLOAT", "total" "FLOAT"}}}]
+ ["postgres" "SELECT \"order\" FROM t"
+ "public" {"public" {"t" {"order" "FLOAT"}}}]
+ ["postgres" "SELECT order FROM t"
+ "public" {"public" {"t" {"order" "FLOAT"}}}]
+ ["postgres" "WITH Orders AS (SELECT 1 AS id) SELECT id FROM ORDERS"
+ "public" {"public" {"unrelated" {"x" "INT"}}}]
+ ["snowflake" "WITH x AS (SELECT * FROM orders) SELECT subtotal FROM x"
+ "PUBLIC" {"PUBLIC" {"ORDERS" {"SUBTOTAL" "FLOAT"}}}]]]
+ (testing sql
+ (is (= "ok" (:status (sql-parsing/validate-query
+ dialect (transpiled-sql dialect sql) default-schema sqlglot-schema))))))))
diff --git a/test/metabase/sql_tools/core_test.clj b/test/metabase/sql_tools/core_test.clj
index 19393a99457..e09473f9452 100644
--- a/test/metabase/sql_tools/core_test.clj
+++ b/test/metabase/sql_tools/core_test.clj
@@ -117,34 +117,16 @@
;; transpile-sql is only implemented for the :sqlglot backend, so these tests bind it directly
;; rather than using test-parser-backends.
-(deftest ^:parallel transpile-sql-snowflake-quotes-identifiers-test
+(deftest ^:parallel transpile-sql-no-added-quoting-test
(binding [sql-tools.settings/*parser-backend-override* :sqlglot]
- (testing "Snowflake should quote identifiers (case-sensitive dialect)"
- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM PUBLIC.users"
- "snowflake" "snowflake")]
- (is (= :success status))
- (is (some? transpiled-sql))
- (is (str/includes? transpiled-sql "\"PUBLIC\""))
- (is (str/includes? transpiled-sql "\"users\""))))))
-
-(deftest ^:parallel transpile-sql-postgres-quotes-identifiers-test
- (binding [sql-tools.settings/*parser-backend-override* :sqlglot]
- (testing "PostgreSQL should quote identifiers (case-sensitive dialect)"
- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM public.users"
- "postgres" "postgres")]
- (is (= :success status))
- (is (some? transpiled-sql))
- (is (str/includes? transpiled-sql "\"public\""))
- (is (str/includes? transpiled-sql "\"users\""))))))
-
-(deftest ^:parallel transpile-sql-mysql-no-quoting-test
- (binding [sql-tools.settings/*parser-backend-override* :sqlglot]
- (testing "MySQL should not add double-quote identifier quoting (not case-sensitive)"
- (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM users"
- "mysql" "mysql")]
- (is (= :success status))
- (is (some? transpiled-sql))
- (is (not (str/includes? transpiled-sql "\"")))))))
+ (testing "unquoted identifiers stay unquoted, so folding dialects resolve them as written"
+ (doseq [dialect ["snowflake" "postgres" "mysql"]]
+ (testing dialect
+ (let [{:keys [status transpiled-sql]} (sql-tools/transpile-sql "SELECT id FROM public.users"
+ dialect dialect)]
+ (is (= :success status))
+ (is (some? transpiled-sql))
+ (is (not (re-find #"[\"`]" transpiled-sql)))))))))
(deftest ^:parallel transpile-sql-multi-statement-rejected-test
(binding [sql-tools.settings/*parser-backend-override* :sqlglot]