Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve quoting and literal handling (#80254)

Nathan Voxlande34951e8b76a
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 6a4c0acf419..1f33d5794af 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -797,8 +797,8 @@
(defmethod sql.qp/inline-value [:bigquery-cloud-sdk String]
[_ s]
- ;; escape single-quotes like Cam's String -> Cam\'s String
- (str \' (str/replace s "'" "\\\\'") \'))
+ ;; escape single-quotes like Cam's String -> Cam\'s String.
+ (sql.u/quote-literal s :backslashes))
(defmethod sql.qp/inline-value [:bigquery-cloud-sdk LocalTime]
[_ t]
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index 5c13a65e1ff..8fd19bbd90a 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -1233,6 +1233,14 @@
(is (= [["2" 1]]
(mt/rows (qp/process-query query)))))))))
+(deftest ^:parallel inline-value-string-test
+ (testing "inlined string literals escape the backslash before the quote"
+ (are [s expected] (= expected (sql.qp/inline-value :bigquery-cloud-sdk s))
+ "Tito's Tacos" "'Tito\\'s Tacos'"
+ "back\\slash" "'back\\\\slash'"
+ "' OR 1 = 1 --" "'\\' OR 1 = 1 --'"
+ "a\\' OR 1 = 1 --" "'a\\\\\\' OR 1 = 1 --'")))
+
(deftest ^:parallel cast-timestamp-to-datetime-if-needed-for-temporal-arithmetic-test
(testing "cast timestamps to datetimes so we can use DATETIME_ADD() if needed for units like month (#21969)"
(is (= ["DATETIME_ADD(CAST(? AS datetime), INTERVAL 3 month)"
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index eaed7124528..a41b335f8c9 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -473,19 +473,21 @@
[_driver _conn role]
;; Since Clickhouse does not truly support prepared statements with protocol-level safety and has no
;; `quote_ident()` function or similar, escape/quote the identifier client-side. The whole value is quoted
- ;; as one identifier, so a role name containing a comma stays a single role. Backslashes are escaped so a
- ;; trailing backslash cannot close the quoted identifier, and interior double-quotes are doubled.
- (let [default-role (driver.sql/default-database-role :clickhouse nil)
- quote-if-needed (fn [role]
- (if (or (and (str/starts-with? role "\"")
- (str/ends-with? role "\""))
- (= role default-role))
+ ;; as one identifier, so a role name containing a comma stays a single role. ClickHouse honors backslash
+ ;; escapes inside a quoted identifier, so `:ansi+backslashes` is the style that applies -- a trailing `\`
+ ;; must not be able to escape the closing quote. HoneySQL's ANSI quoting only doubles the quote character,
+ ;; which is why this goes through [[sql.u/quote-identifier]] rather than [[sql.u/quote-name]].
+ (let [default-role (driver.sql/default-database-role :clickhouse nil)
+ ;; a value the caller already quoted is unwrapped first, so its contents are escaped like any other.
+ ;; It takes two characters to be a wrapped value -- a lone `"` is a one-character role name.
+ unwrapped (if (and (>= (count role) 2)
+ (str/starts-with? role "\"")
+ (str/ends-with? role "\""))
+ (subs role 1 (dec (count role)))
+ role)]
+ (format "SET ROLE %s" (if (= role default-role)
role
- (str \" role \")))
- escape-ident #(-> %
- (str/replace "\\" "\\\\")
- (str/replace #"(?!^)\"(?<!$)" "\"\""))]
- (format "SET ROLE %s" (-> role quote-if-needed escape-ident))))
+ (sql.u/quote-identifier unwrapped :ansi+backslashes)))))
(defmethod driver/set-role! :clickhouse
[driver ^Connection conn role]
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
index 692707c986f..20bd4b86111 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
@@ -675,6 +675,10 @@
;; _
:else (.getObject rs i)))))
+(defmethod sql.qp/inline-value [:clickhouse String]
+ [_driver ^String s]
+ (sql.u/quote-literal s :backslashes))
+
(defmethod sql.qp/inline-value [:clickhouse LocalDate]
[_ t]
(format "'%s'" (t/format "yyyy-MM-dd" t)))
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 0b74c51e27f..5e4b978ee20 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -3,6 +3,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.clickhouse-test]}}}}}}
(:require
[clojure.java.jdbc :as jdbc]
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.clickhouse :as clickhouse]
@@ -11,10 +12,13 @@
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib-be.core :as lib-be]
[metabase.lib.card :as lib.card]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.test :as qp]
[metabase.sync.sync :as sync]
@@ -53,6 +57,61 @@
"" [] ; blank -> [], so :key-columns stays schema-valid
nil [])))
+(deftest ^:parallel inline-value-string-test
+ (testing "inlined string literals escape the backslash before the quote"
+ ;; ClickHouse treats `\` as an escape character inside a string literal, so doubling `'` alone (the default
+ ;; `[:sql String]` behaviour) lets a value like `a\'` close the literal early and run the rest as SQL.
+ (are [s expected] (= expected (sql.qp/inline-value :clickhouse s))
+ "Tito's Tacos" "'Tito\\'s Tacos'"
+ "back\\slash" "'back\\\\slash'"
+ "' OR 1 = 1 --" "'\\' OR 1 = 1 --'"
+ "a\\' OR 1 = 1 --" "'a\\\\\\' OR 1 = 1 --'")))
+
+(def ^:private breakout-payload
+ "A value that closes a ClickHouse string literal early unless its backslash is escaped: `a\\' or 1=1 -- `."
+ "a\\' or 1=1 -- ")
+
+(def ^:private escaped-breakout-payload
+ "[[breakout-payload]] correctly escaped: `\\` is doubled, so the quote that follows is a genuinely escaped quote
+ and the payload stays inside the literal."
+ "a\\\\\\' or 1=1 -- ")
+
+;;; `contains` / `starts-with` / `ends-with` are an additional carrier for the escaping defect above, and a
+;;; very common one. On the generic SQL path they compile to `LIKE <pattern>`, and `sql.qp/generate-pattern` runs
+;;; `escape-like-pattern` on the value first -- which doubles `\` and so happens to neutralise this payload shape.
+;;; ClickHouse overrides all three to its native scalar functions instead, so `generate-pattern` never runs and the
+;;; value reaches ordinary function-argument position unescaped. Only [[sql.qp/inline-value]] stands between it and
+;;; the SQL text.
+(deftest string-filter-inline-escaping-test
+ ;; no ClickHouse server needed -- this only compiles the query -- but the QP pipeline reads the app DB
+ (mt/initialize-if-needed! :db)
+ (testing "a string filter value cannot break out of the literal when compiled with inline parameters"
+ (let [mp (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:database {:engine :clickhouse
+ :dbms-version {:version "24.4" :semantic-version {:major 24 :minor 4}}}})
+ venues (lib.metadata/table mp (meta/id :venues))
+ name-col (lib.metadata/field mp (meta/id :venues :name))
+ compile! (fn [filter-clause]
+ (:query (qp.compile/compile-with-inline-parameters
+ (-> (lib/query mp venues)
+ (lib/filter filter-clause)))))]
+ (doseq [[msg filter-clause expected]
+ [["contains" (lib/contains name-col breakout-payload)
+ (format "`positionUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]
+ ["starts-with" (lib/starts-with name-col breakout-payload)
+ (format "`startsWithUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]
+ ["ends-with" (lib/ends-with name-col breakout-payload)
+ (format "`endsWithUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]
+ ["case-insensitive contains" (lib/ignore-case (lib/contains name-col breakout-payload))
+ (format "`positionCaseInsensitiveUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]
+ ["case-insensitive starts-with" (lib/ignore-case (lib/starts-with name-col breakout-payload))
+ (format "`lowerUTF8`('%s')" escaped-breakout-payload)]
+ ["case-insensitive ends-with" (lib/ignore-case (lib/ends-with name-col breakout-payload))
+ (format "`lowerUTF8`('%s')" escaped-breakout-payload)]]]
+ (testing msg
+ (is (str/includes? (compile! filter-clause) expected)))))))
+
(deftest ^:parallel clickhouse-version
(mt/test-driver :clickhouse
(t2.with-temp/with-temp
@@ -622,4 +681,7 @@
"\"x\"; SELECT sleep(10); --\"" "SET ROLE \"x\"\"; SELECT sleep(10); --\""
;; a trailing backslash is escaped so it cannot close the quoted identifier
"foo\\" "SET ROLE \"foo\\\\\""
- "a\\\"b" "SET ROLE \"a\\\\\"\"b\""))
+ "a\\\"b" "SET ROLE \"a\\\\\"\"b\""
+ ;; a lone double-quote is a one-character role name, not an already-quoted empty one -- it must still come
+ ;; out as a terminated identifier
+ "\"" "SET ROLE \"\"\"\""))
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index d76659d7ea8..4cb82f8a33e 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -778,6 +778,15 @@
[driver t]
(sql.qp/->honeysql driver (t/offset-date-time t)))
+;;; Snowflake treats `\` inside a string literal as an escape introducer -- `\'` is an escaped single quote and `\\`
+;;; an escaped backslash (https://docs.snowflake.com/en/sql-reference/data-types-text). The generic `[:sql String]`
+;;; implementation only doubles the single quotes, which leaves a value containing `\'` (or ending in `\`) free to
+;;; terminate the literal one quote early. Snowflake accepts both escape forms, so use `:ansi+backslashes`: doubling
+;;; the backslash means it can never escape our closing quote.
+(defmethod sql.qp/inline-value [:snowflake String]
+ [_driver ^String s]
+ (sql.u/quote-literal s :ansi+backslashes))
+
(defmethod driver/table-rows-seq :snowflake
[driver database table]
(driver-api/with-metadata-provider (u/the-id database)
@@ -1171,7 +1180,11 @@
255)
(defn get-string-filter-arg
- "Generate the argument to match in the string filters. It's based on sql.qp/generate-pattern."
+ "Generate the argument to match in the string filters. It's based on sql.qp/generate-pattern.
+
+ Unlike `sql.qp/generate-pattern` this does no escaping: these filters compile to Snowflake's native scalar
+ functions rather than `LIKE`, so there is no pattern to escape, and the value has to stay verbatim to bind
+ correctly as a `?` parameter. Escaping the value for an inline compile is [[sql.qp/inline-value]]'s job."
[driver
[type opts val :as arg]
{:keys [case-sensitive] :or {case-sensitive true} :as _options}]
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 8ec8bbaa310..0ae15fa74ed 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -29,6 +29,7 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.notebook-helpers :as notebook-helpers]
+ [metabase.query-processor.compile :as qp.compile]
^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
[metabase.secrets.core :as secret]
@@ -1733,6 +1734,62 @@
(is (str/includes? (-> result :data :native_form :query) exp-filter))
(is (= exp-rows (mt/rows result)))))))))
+(def ^:private breakout-payload
+ "A value that closes a Snowflake string literal early unless its backslash is escaped: `a\\' or 1=1 -- `.
+ Kept all-lowercase so the case-insensitive filters, which lower-case the value, expect the same literal."
+ "a\\' or 1=1 -- ")
+
+(def ^:private escaped-breakout-payload
+ "[[breakout-payload]] correctly escaped: the `\\` is doubled so it cannot escape anything, and the `'` is doubled,
+ so the payload stays inside the literal."
+ "a\\\\'' or 1=1 -- ")
+
+(deftest ^:parallel inline-value-string-test
+ (testing "inlined string literals escape the backslash as well as the quote"
+ ;; Snowflake treats `\` as an escape character inside a string literal, so doubling `'` alone (the default
+ ;; `[:sql String]` behaviour) lets a value like `a\'` close the literal early and run the rest as SQL.
+ (are [s expected] (= expected (sql.qp/inline-value :snowflake s))
+ "Tito's Tacos" "'Tito''s Tacos'" ; 'Tito''s Tacos'
+ "'" "''''" ; ''''
+ "back\\slash" "'back\\\\slash'" ; 'back\\slash'
+ "trailing\\" "'trailing\\\\'" ; 'trailing\\'
+ breakout-payload (str \' escaped-breakout-payload \'))))
+
+;;; `contains` / `starts-with` / `ends-with` are an additional carrier for the escaping defect above, and a
+;;; very common one. On the generic SQL path they compile to `LIKE <pattern>`, and `sql.qp/generate-pattern` runs
+;;; `escape-like-pattern` on the value first -- which doubles `\` and so happens to neutralise this payload shape.
+;;; Snowflake overrides all three to its native scalar functions instead, so `generate-pattern` never runs and the
+;;; value reaches ordinary function-argument position unescaped. Only [[sql.qp/inline-value]] stands between it and
+;;; the SQL text.
+(deftest string-filter-inline-escaping-test
+ ;; no Snowflake warehouse needed -- this only compiles the query -- but the QP pipeline reads the app DB
+ (mt/initialize-if-needed! :db)
+ (testing "a string filter value cannot break out of the literal when compiled with inline parameters"
+ (let [mp (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:database {:engine :snowflake, :details {:db "test-data"}}})
+ venues (lib.metadata/table mp (meta/id :venues))
+ name-col (lib.metadata/field mp (meta/id :venues :name))
+ compile! (fn [filter-clause]
+ (:query (qp.compile/compile-with-inline-parameters
+ (-> (lib/query mp venues)
+ (lib/filter filter-clause)))))]
+ (doseq [[msg filter-clause expected]
+ [["contains" (lib/contains name-col breakout-payload)
+ (format "CONTAINS(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]
+ ["starts-with" (lib/starts-with name-col breakout-payload)
+ (format "STARTSWITH(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]
+ ["ends-with" (lib/ends-with name-col breakout-payload)
+ (format "ENDSWITH(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]
+ ["case-insensitive contains" (lib/ignore-case (lib/contains name-col breakout-payload))
+ (format "CONTAINS(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]
+ ["case-insensitive starts-with" (lib/ignore-case (lib/starts-with name-col breakout-payload))
+ (format "STARTSWITH(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]
+ ["case-insensitive ends-with" (lib/ignore-case (lib/ends-with name-col breakout-payload))
+ (format "ENDSWITH(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]]]
+ (testing msg
+ (is (str/includes? (compile! filter-clause) expected)))))))
+
(deftest snowflake-collate-comparison-test
(mt/test-driver :snowflake
(let [mp (mt/metadata-provider)
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index af1f4bb6357..f5be6ccc7e3 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -887,6 +887,21 @@
"UTC"
offset)))
+(defn- utf8-string-literal
+ "A MySQL string literal for `s`, written as a character set introducer plus a hex literal so its value does not
+ depend on the session's `NO_BACKSLASH_ESCAPES` setting. This has to be a *literal* rather than
+ `CONVERT(UNHEX(...) USING utf8mb4)`: a literal keeps coercibility `COERCIBLE` and so takes on the collation of
+ whatever it is compared against, while `CONVERT` is `IMPLICIT` and makes every comparison against a column whose
+ collation differs from the connection's fail with `Illegal mix of collations`."
+ [^String s]
+ (format "_utf8mb4 X'%s'" (codecs/bytes->hex (.getBytes s "UTF-8"))))
+
+(defmethod sql.qp/inline-value [:mysql String]
+ [_driver ^String s]
+ ;; MySQL's interpretation of backslashes in quoted literals depends on `NO_BACKSLASH_ESCAPES`. Use a hex literal so
+ ;; both the safety and the value of the resulting string are independent of the session's SQL mode.
+ (utf8-string-literal s))
+
(defmethod sql.qp/inline-value [:mysql OffsetTime]
[_ t]
;; MySQL doesn't support timezone offsets in literals so pass in a local time literal wrapped in a call to convert
@@ -1360,7 +1375,9 @@
(defn- utf8-string-expr
"A MySQL expression evaluating to the string `s`, hex-encoded so there is nothing to escape and SQL injection is
impossible. Index names are unvalidated free-form user input and `sql.u/escape-sql` is explicitly not safe for that
- (a backslash defeats its quote-doubling, and its escaping is session-dependent), so we use the hex pattern instead."
+ (a backslash defeats its quote-doubling, and its escaping is session-dependent), so we use the hex pattern instead.
+ Use [[utf8-string-literal]] instead anywhere the result is compared against a column -- this one is `IMPLICIT` and
+ would clash with the column's collation."
[^String s]
(format "CONVERT(UNHEX('%s') USING utf8mb4)" (codecs/bytes->hex (.getBytes s "UTF-8"))))
diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.clj
index 1c6b4c406e5..afdee820d11 100644
--- a/src/metabase/driver/sql/util.clj
+++ b/src/metabase/driver/sql/util.clj
@@ -104,13 +104,40 @@
:else
(recur (conj already-seen alias) (conj acc [col alias]) more)))))
+(defn- escape-quoted
+ "Shared implementation of [[escape-sql]] and [[escape-identifier]]. Escapes `quote-char` -- and, for the styles that
+ call for it, the backslash -- so `s` can be spliced between a pair of `quote-char`s without terminating early."
+ ^String [^String s ^Character quote-char escape-style]
+ (let [q (str quote-char)
+ doubled (str quote-char quote-char)]
+ (case escape-style
+ :ansi (str/replace s q doubled)
+ :backslashes (-> s
+ (str/replace "\\" "\\\\")
+ (str/replace q (str "\\" q)))
+ :ansi+backslashes (-> s
+ (str/replace "\\" "\\\\")
+ (str/replace q doubled)))))
+
;;; TODO (Cam 2026-04-27) -- rename this to `escape-single-quotes` to make it clearer what we're escaping
(defn escape-sql
- "Escape single quotes in a SQL string. `escape-style` is either `:ansi` (escape a single quote with two single quotes)
- or `:backslashes` (escape a single quote with a backslash).
+ "Escape single quotes in a SQL string. `escape-style` is one of
+
+ * `:ansi` -- escape a single quote with two single quotes. Correct only on engines that do *not* treat `\\` as an
+ escape character inside a string literal (Postgres with `standard_conforming_strings`, H2, SQL Server, Oracle...).
+
+ * `:backslashes` -- escape the backslash, then escape a single quote with a backslash. For engines that only
+ understand the backslash form (BigQuery, Presto...).
+
+ * `:ansi+backslashes` -- escape the backslash, then escape a single quote with two single quotes. For engines that
+ always accept both forms, such as Snowflake: doubling the backslash means no `\\` can ever escape our closing quote,
+ and doubling the quote terminates the literal where we intended. Do not use this on engines whose backslash
+ handling is configurable: it is injection-safe in either mode, but does not preserve the value when backslashes
+ are treated literally.
- (escape-sql \"Tito's Tacos\" :ansi) ; -> \"Tito''s Tacos\"
- (escape-sql \"Tito's Tacos\" :backslashes) ; -> \"Tito\\'s Tacos\"
+ (escape-sql \"Tito's Tacos\" :ansi) ; -> \"Tito''s Tacos\"
+ (escape-sql \"Tito's Tacos\" :backslashes) ; -> \"Tito\\'s Tacos\"
+ (escape-sql \"Tito's Tacos\" :ansi+backslashes) ; -> \"Tito''s Tacos\"
!!!! VERY IMPORTANT !!!!
@@ -120,14 +147,10 @@
encode the strings as hex and splice in something along the lines of `utf8_string(hex_decode(<hex-string>))`
instead. This is intended only for escaping trusted strings, or for generating the SQL equivalent version of an MBQL
query for debugging purposes or powering the 'convert to SQL' feature."
- {:arglists '([s :ansi] [s :backslashes])}
+ {:arglists '([s :ansi] [s :backslashes] [s :ansi+backslashes])}
^String [^String s escape-style]
(when s
- (case escape-style
- :ansi (str/replace s "'" "''")
- :backslashes (-> s
- (str/replace "\\" "\\\\")
- (str/replace "'" "\\'")))))
+ (escape-quoted s \' escape-style)))
(defn quote-literal
"Wrap `s` in single quotes as a SQL string literal, escaping embedded quotes per `escape-style`.
@@ -139,7 +162,41 @@
^String [^String s escape-style]
(when s
(case escape-style
- (:ansi :backslashes) (str \' (escape-sql s escape-style) \'))))
+ (:ansi :backslashes :ansi+backslashes) (str \' (escape-sql s escape-style) \'))))
+
+(defn escape-identifier
+ "Escape `s` so it can be spliced between a pair of double quotes as a SQL identifier. `escape-style` is the same
+ vocabulary as [[escape-sql]], applied to `\"` rather than `'`:
+
+ * `:ansi` -- double an embedded `\"`. Correct on engines that do *not* treat `\\` as an escape character inside a
+ quoted identifier.
+
+ * `:ansi+backslashes` -- escape the backslash as well, so a trailing `\\` cannot escape our closing quote. Needed on
+ engines that honour backslash escapes inside quoted identifiers, such as ClickHouse.
+
+ `:backslashes` behaves as it does in [[escape-sql]], applied to `\"`; no current caller needs it.
+
+ (escape-identifier \"a\\\"b\" :ansi) ; -> \"a\\\"\\\"b\"
+ (escape-identifier \"trailing\\\\\" :ansi+backslashes) ; -> \"trailing\\\\\\\\\"
+
+ Prefer [[quote-name]] when the driver's `sql.qp/quote-style` already escapes correctly -- it goes through HoneySQL
+ and knows about backtick dialects. Reach for this only when hand-building SQL text for an engine whose escaping
+ HoneySQL does not cover.
+
+ For user input, *ALWAYS* prefer a query parameter or the engine's own `quote_ident()` where one exists."
+ {:arglists '([s :ansi] [s :ansi+backslashes])}
+ ^String [^String s escape-style]
+ (when s
+ (escape-quoted s \" escape-style)))
+
+(defn quote-identifier
+ "Wrap `s` in double quotes as a SQL identifier, escaping embedded quotes per `escape-style`. See
+ [[escape-identifier]] for the escape styles and for when to prefer [[quote-name]].
+
+ (quote-identifier \"a\\\"b\" :ansi) ; -> \"\\\"a\\\"\\\"b\\\"\""
+ ^String [^String s escape-style]
+ (when s
+ (str \" (escape-identifier s escape-style) \")))
(defn validate-convert-timezone-args
"Validate the arguments of convert-timezone.
diff --git a/src/metabase/warehouse_schema/field_values/distinct_batch.clj b/src/metabase/warehouse_schema/field_values/distinct_batch.clj
index ab3080e60d7..24535281d48 100644
--- a/src/metabase/warehouse_schema/field_values/distinct_batch.clj
+++ b/src/metabase/warehouse_schema/field_values/distinct_batch.clj
@@ -3,11 +3,16 @@
For each field in the input set, builds a flat per-field arm of the shape
- SELECT '<field-name>' AS field_name, <cast-to-text>(col) AS value_out
+ SELECT <arm-ordinal> AS field_idx, <cast-to-text>(col) AS value_out
FROM <table>
GROUP BY <cast-to-text>(col)
<driver-correct LIMIT clause>
+ Arms are tagged with their ordinal in the batch rather than with the field's name. The name comes from the
+ warehouse verbatim, and an inline string literal built from it is a SQL injection sink on any engine whose
+ string lexer we don't escape for; an integer ordinal is unforgeable. It also means two fields that
+ happen to share a name stay distinct.
+
Per-arm DISTINCT semantics come from `GROUP BY` rather than `SELECT DISTINCT` — both are
equivalent here, but `GROUP BY` composes cleanly with every driver's
[[sql.qp/apply-top-level-clause]] `:limit` transform (notably SQL Server's `TOP N` rewrite,
@@ -27,6 +32,7 @@
chain-filter → params.field-values → field → field-values) would form a cycle if these
requires lived in `metabase.warehouse-schema.models.field-values`."
(:require
+ [clojure.string :as str]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.query-processor :as qp]
[metabase.util :as u]
@@ -81,15 +87,15 @@
(sql.qp/->honeysql driver [::sql.qp/cast-to-text {} (h2x/identifier :field col-name)]))
(defn- build-arm
- "HoneySQL for one UNION arm."
- [driver table field]
+ "HoneySQL for one UNION arm. `idx` is the field's position in the batch and becomes the arm's tag."
+ [driver table field idx]
(let [cast-expr (cast-to-text-honeysql driver (:name field))
- inner {:select [[[:inline (:name field)] :field_name]
+ inner {:select [[[:inline idx] :field_idx]
[cast-expr :value_out]]
;; `:from` wraps the identifier expression in an extra vector — `[[expr]]` so HoneySQL
;; treats it as a single table expression rather than `[table alias …]`.
:from [[(table-honeysql driver table)]]
- ;; Only `cast-expr` goes in `:group-by`. The `[:inline (:name field)]` tag is a literal in
+ ;; Only `cast-expr` goes in `:group-by`. The `[:inline idx]` tag is a literal in
;; the SELECT list and needs no GROUP BY entry.
:group-by [cast-expr]}
limited (sql.qp/apply-top-level-clause driver :limit inner {:limit field-values/*distinct-limit*})]
@@ -104,11 +110,20 @@
(defn- build-union
"Build the full HoneySQL form for one batch of fields. Single field → no UNION wrapper."
[driver table fields]
- (let [arms (mapv #(build-arm driver table %) fields)]
+ (let [arms (into [] (map-indexed #(build-arm driver table %2 %1)) fields)]
(if (= 1 (count arms))
(first arms)
{:union-all arms})))
+(defn- decode-arm-idx
+ "Coerce the `field_idx` tag a driver returned back to a Clojure integer. Drivers disagree on the type they
+ hand back for an inline integer literal (`Long` on most, `BigDecimal` on Oracle/Snowflake, …), so normalize
+ rather than assume. Returns `nil` for anything unrecognizable."
+ [x]
+ (cond
+ (number? x) (long x)
+ (string? x) (try (Long/parseLong (str/trim x)) (catch NumberFormatException _ nil))))
+
(defn run-distinct-batch
"Execute one UNION ALL query covering `fields` from one `table`. The caller is responsible for
ensuring `(count fields)` ≤ [[*batch-size*]] (the per-query arm cap) and that all `fields`
@@ -123,19 +138,23 @@
[table fields]
(let [db-id (:db_id table)
driver (:engine (t2/select-one :model/Database :id db-id))
+ fields (vec fields)
hsql (build-union driver table fields)
[sql & params] (sql.qp/format-honeysql driver hsql)
result (qp/process-query
{:database db-id
:type :native
:native {:query sql, :params (vec params)}})
- rows (-> result :data :rows)
- by-name (into {} (map (juxt :name identity)) fields)]
- (reduce (fn [acc [field-name value-str]]
- (let [field (get by-name field-name)
- v (decode-value (:base_type field) value-str)]
- (-> acc
- (update-in [(:id field) :values] (fnil conj []) v)
- (update-in [(:id field) :raw-count] (fnil inc 0)))))
+ rows (-> result :data :rows)]
+ (reduce (fn [acc [arm-idx value-str]]
+ ;; Arms are tagged by ordinal rather than by name: the name is warehouse-controlled and has no
+ ;; business being in the query text as a literal, and ordinals also keep two
+ ;; identically-named fields in the same batch from collapsing into one.
+ (if-let [field (get fields (decode-arm-idx arm-idx))]
+ (let [v (decode-value (:base_type field) value-str)]
+ (-> acc
+ (update-in [(:id field) :values] (fnil conj []) v)
+ (update-in [(:id field) :raw-count] (fnil inc 0))))
+ acc))
(into {} (map (fn [f] [(:id f) {:values [] :raw-count 0}])) fields)
rows)))
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 775c78f8faf..bb860aa70d0 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -58,6 +58,23 @@
(binding [sync-util/*log-exceptions-and-continue?* false]
(mt/with-test-user :rasta (thunk)))))
+(deftest ^:parallel inline-value-string-test
+ (testing "inlined strings use a representation that is independent of the session's backslash-escaping mode"
+ (let [expected (fn [^String s]
+ (format "_utf8mb4 X'%s'" (codecs/bytes->hex (.getBytes s "UTF-8"))))]
+ (are [s] (= (expected s) (sql.qp/inline-value :mysql s))
+ "Tito's Tacos"
+ "back\\slash"
+ "a\\' OR 1 = 1 --"
+ ""
+ "\u00e9\ud83c\udf63")))
+ (testing "compiling a query with inline parameters"
+ (binding [driver/*compile-with-inline-parameters* true]
+ (is (= ["SELECT * FROM `venues` WHERE `venues`.`name` = _utf8mb4 X'615c27204f522031203d2031202d2d'"]
+ (sql.qp/format-honeysql :mysql {:select [:*]
+ :from [[:venues]]
+ :where [:= :venues/name "a\\' OR 1 = 1 --"]}))))))
+
(deftest all-zero-dates-test
(mt/test-driver :mysql
(testing (str "MySQL allows 0000-00-00 dates, but JDBC does not; make sure that MySQL is converting them to NULL "
diff --git a/test/metabase/driver/sql/util_test.clj b/test/metabase/driver/sql/util_test.clj
index b2c57782a35..a66337842f2 100644
--- a/test/metabase/driver/sql/util_test.clj
+++ b/test/metabase/driver/sql/util_test.clj
@@ -80,7 +80,16 @@
"\\\\\\\\' OR 1 = 1 --" "\\\\\\\\\\\\\\\\\\' OR 1 = 1 --"
"\\\\' OR 1 = 1 --" "\\\\\\\\\\' OR 1 = 1 --"
"\\' OR 1 = 1 --" "\\\\\\' OR 1 = 1 --"
- "' OR 1 = 1 --" "\\' OR 1 = 1 --"}}
+ "' OR 1 = 1 --" "\\' OR 1 = 1 --"}
+
+ ;; doubles the backslash *and* the quote, so the literal terminates where we intended whether or not
+ ;; the engine treats `\` as an escape character
+ :ansi+backslashes
+ {"Tito's Tacos" "Tito''s Tacos"
+ "\\\\\\\\' OR 1 = 1 --" "\\\\\\\\\\\\\\\\'' OR 1 = 1 --"
+ "\\\\' OR 1 = 1 --" "\\\\\\\\'' OR 1 = 1 --"
+ "\\' OR 1 = 1 --" "\\\\'' OR 1 = 1 --"
+ "' OR 1 = 1 --" "'' OR 1 = 1 --"}}
[s expected] s->expected]
(testing escape-strategy
@@ -95,9 +104,49 @@
(testing ":backslashes escape style"
(is (= "'Tito\\'s Tacos'"
(sql.u/quote-literal "Tito's Tacos" :backslashes))))
+ (testing ":ansi+backslashes escape style"
+ (is (= "'Tito''s Tacos'"
+ (sql.u/quote-literal "Tito's Tacos" :ansi+backslashes)))
+ (is (= "'a\\\\'' OR 1 = 1 --'"
+ (sql.u/quote-literal "a\\' OR 1 = 1 --" :ansi+backslashes))))
(testing "empty string"
(is (= "''" (sql.u/quote-literal "" :ansi)))))
+(deftest ^:parallel escape-identifier-test
+ (doseq [[escape-strategy s->expected]
+ {:ansi
+ {"role" "role"
+ "a\"b" "a\"\"b"
+ "trailing\\" "trailing\\"
+ "a\\\"b" "a\\\"\"b"}
+
+ ;; doubles the backslash *and* the quote, so the identifier terminates where we intended whether or not
+ ;; the engine treats `\` as an escape character. This is what ClickHouse needs.
+ :ansi+backslashes
+ {"role" "role"
+ "a\"b" "a\"\"b"
+ "trailing\\" "trailing\\\\"
+ "a\\\"b" "a\\\\\"\"b"}}
+
+ [s expected] s->expected]
+ (testing escape-strategy
+ (testing (pr-str s)
+ (is (= expected
+ (sql.u/escape-identifier s escape-strategy)))))))
+
+(deftest ^:parallel quote-identifier-test
+ (testing "wraps in double quotes and doubles interior quotes"
+ (is (= "\"role\"" (sql.u/quote-identifier "role" :ansi)))
+ (is (= "\"a\"\"b\"" (sql.u/quote-identifier "a\"b" :ansi))))
+ (testing ":ansi+backslashes also escapes backslashes"
+ (testing "a trailing backslash cannot escape our closing quote"
+ (is (= "\"trailing\\\\\"" (sql.u/quote-identifier "trailing\\" :ansi+backslashes))))
+ (is (= "\"a\\\\\"\"b\"" (sql.u/quote-identifier "a\\\"b" :ansi+backslashes))))
+ (testing "empty string"
+ (is (= "\"\"" (sql.u/quote-identifier "" :ansi))))
+ (testing "nil"
+ (is (nil? (sql.u/quote-identifier nil :ansi)))))
+
;;; Ok to hardcode driver names in the tests below because they're for general util functions and not something that
;;; needs to be run against all supported drivers
diff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.clj
index eb7f4613e5c..cfa88c11d89 100644
--- a/test/metabase/warehouse_schema/models/field_values_test.clj
+++ b/test/metabase/warehouse_schema/models/field_values_test.clj
@@ -3,10 +3,13 @@
the [[metabase.warehouse-schema.models.field-values]] namespace."
(:require
[clojure.java.jdbc :as jdbc]
+ [clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.query-processor :as qp]
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -632,6 +635,57 @@
(testing "Unknown base-type → string passthrough"
(is (= "anything" (#'distinct-batch/decode-value :type/SomeMadeUpType "anything")))))
+;;; Column names are harvested verbatim from the warehouse by `describe-table` / `describe-fields`, so they are
+;;; attacker-controlled for anyone who can create a column in a synced schema. They must never reach the generated
+;;; SQL as a string literal -- only as a quoted identifier.
+(def ^:private sql-injection-field-name
+ "a\\' AS `field_name`, (SELECT @@version) AS `value_out` FROM mysql.db LIMIT 1) AS `_arm` -- ")
+
+(deftest ^:parallel build-union-tags-arms-by-ordinal-test
+ (testing "arms are tagged with their ordinal, not with the field name"
+ (let [fields [{:name "state" :base_type :type/Text}
+ {:name "source" :base_type :type/Text}]]
+ (doseq [[driver q] {:h2 "\"", :postgres "\"", :mysql "`"}]
+ (testing driver
+ (let [sql (first (sql.qp/format-honeysql driver (#'distinct-batch/build-union driver {:name "t"} fields)))]
+ (is (str/includes? sql (str "0 AS " q "field_idx" q)))
+ (is (str/includes? sql (str "1 AS " q "field_idx" q)))
+ (is (not (str/includes? sql "'"))
+ (str "no SQL string literal is generated at all. Got: " sql))))))))
+
+(deftest ^:parallel build-union-field-name-cannot-break-out-of-sql-test
+ (testing "a warehouse column name crafted to break out of a string literal stays inert"
+ ;; On MySQL `\'` is a second way to write a quote inside a literal, so the old `[:inline (:name field)]` tag
+ ;; -- escaped only by doubling `'` -- closed early and the rest of the column name ran as SQL. Post-fix
+ ;; there is no tag literal at all, and the name reaches the query only as a quoted identifier (which is
+ ;; where the column genuinely is), with the identifier quote character doubled.
+ (let [fields [{:name sql-injection-field-name :base_type :type/Text}]
+ sql (first (sql.qp/format-honeysql :mysql (#'distinct-batch/build-union :mysql {:name "t"} fields)))
+ quoted-name (str "`" (str/replace sql-injection-field-name "`" "``") "`")]
+ (is (= (str "SELECT * FROM ("
+ "SELECT 0 AS `field_idx`, CAST(" quoted-name " AS char) AS `value_out` "
+ "FROM `t` "
+ "GROUP BY CAST(" quoted-name " AS char) "
+ "LIMIT 1000) AS `_arm`")
+ sql)))))
+
+(deftest run-distinct-batch-demuxes-by-arm-ordinal-test
+ (testing "rows are mapped back to fields by arm ordinal, not by name"
+ ;; `idx_unique_field` only makes `name` unique per (table, parent) pair, so a table can legitimately hold two
+ ;; fields called `dupe` as long as one of them is nested. Keying the demux by name collapsed both arms onto
+ ;; whichever field won the `by-name` lookup; the ordinal keeps them apart.
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table {table-id :id :as table} {:db_id db-id :name "t"}
+ :model/Field {parent-id :id} {:table_id table-id :name "json_col" :base_type :type/JSON}
+ :model/Field {f1-id :id} {:table_id table-id :name "dupe" :base_type :type/Text}
+ :model/Field {f2-id :id} {:table_id table-id :name "dupe" :base_type :type/Integer
+ :parent_id parent-id}]
+ (let [fields [(t2/select-one :model/Field :id f1-id) (t2/select-one :model/Field :id f2-id)]]
+ (with-redefs [qp/process-query (fn [_query] {:data {:rows [[0 "a"] [0 "b"] [1 "42"]]}})]
+ (is (= {f1-id {:values ["a" "b"] :raw-count 2}
+ f2-id {:values [42] :raw-count 1}}
+ (distinct-batch/run-distinct-batch table fields))))))))
+
(deftest ^:mb/driver-tests run-distinct-batch-integration-test
(testing "run-distinct-batch returns correct distinct values for each field"
(mt/test-drivers (sql-test-drivers)