The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve quoting and literal handling (#80254)
Nathan Voxlande34951e8b76a
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.cljindex 6a4c0acf419..1f33d5794af 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj@@ -797,8 +797,8 @@(defmethod sql.qp/inline-value [:bigquery-cloud-sdk String][_ s]- ;; escape single-quotes like Cam's String -> Cam\'s String- (str \' (str/replace s "'" "\\\\'") \'))+ ;; escape single-quotes like Cam's String -> Cam\'s String.+ (sql.u/quote-literal s :backslashes))(defmethod sql.qp/inline-value [:bigquery-cloud-sdk LocalTime][_ t]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex 5c13a65e1ff..8fd19bbd90a 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -1233,6 +1233,14 @@(is (= [["2" 1]](mt/rows (qp/process-query query)))))))))+(deftest ^:parallel inline-value-string-test+ (testing "inlined string literals escape the backslash before the quote"+ (are [s expected] (= expected (sql.qp/inline-value :bigquery-cloud-sdk s))+ "Tito's Tacos" "'Tito\\'s Tacos'"+ "back\\slash" "'back\\\\slash'"+ "' OR 1 = 1 --" "'\\' OR 1 = 1 --'"+ "a\\' OR 1 = 1 --" "'a\\\\\\' OR 1 = 1 --'")))+(deftest ^:parallel cast-timestamp-to-datetime-if-needed-for-temporal-arithmetic-test(testing "cast timestamps to datetimes so we can use DATETIME_ADD() if needed for units like month (#21969)"(is (= ["DATETIME_ADD(CAST(? AS datetime), INTERVAL 3 month)"diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex eaed7124528..a41b335f8c9 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -473,19 +473,21 @@[_driver _conn role];; Since Clickhouse does not truly support prepared statements with protocol-level safety and has no;; `quote_ident()` function or similar, escape/quote the identifier client-side. The whole value is quoted- ;; as one identifier, so a role name containing a comma stays a single role. Backslashes are escaped so a- ;; trailing backslash cannot close the quoted identifier, and interior double-quotes are doubled.- (let [default-role (driver.sql/default-database-role :clickhouse nil)- quote-if-needed (fn [role]- (if (or (and (str/starts-with? role "\"")- (str/ends-with? role "\""))- (= role default-role))+ ;; as one identifier, so a role name containing a comma stays a single role. ClickHouse honors backslash+ ;; escapes inside a quoted identifier, so `:ansi+backslashes` is the style that applies -- a trailing `\`+ ;; must not be able to escape the closing quote. HoneySQL's ANSI quoting only doubles the quote character,+ ;; which is why this goes through [[sql.u/quote-identifier]] rather than [[sql.u/quote-name]].+ (let [default-role (driver.sql/default-database-role :clickhouse nil)+ ;; a value the caller already quoted is unwrapped first, so its contents are escaped like any other.+ ;; It takes two characters to be a wrapped value -- a lone `"` is a one-character role name.+ unwrapped (if (and (>= (count role) 2)+ (str/starts-with? role "\"")+ (str/ends-with? role "\""))+ (subs role 1 (dec (count role)))+ role)]+ (format "SET ROLE %s" (if (= role default-role)role- (str \" role \")))- escape-ident #(-> %- (str/replace "\\" "\\\\")- (str/replace #"(?!^)\"(?<!$)" "\"\""))]- (format "SET ROLE %s" (-> role quote-if-needed escape-ident))))+ (sql.u/quote-identifier unwrapped :ansi+backslashes)))))(defmethod driver/set-role! :clickhouse[driver ^Connection conn role]diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.cljindex 692707c986f..20bd4b86111 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj@@ -675,6 +675,10 @@;; _:else (.getObject rs i)))))+(defmethod sql.qp/inline-value [:clickhouse String]+ [_driver ^String s]+ (sql.u/quote-literal s :backslashes))+(defmethod sql.qp/inline-value [:clickhouse LocalDate][_ t](format "'%s'" (t/format "yyyy-MM-dd" t)))diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 0b74c51e27f..5e4b978ee20 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -3,6 +3,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.clickhouse-test]}}}}}}(:require[clojure.java.jdbc :as jdbc]+ [clojure.string :as str][clojure.test :refer :all][metabase.driver :as driver][metabase.driver.clickhouse :as clickhouse]@@ -11,10 +12,13 @@[metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.sql.query-processor :as sql.qp][metabase.lib-be.core :as lib-be][metabase.lib.card :as lib.card][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu][metabase.query-processor.compile :as qp.compile][metabase.query-processor.test :as qp][metabase.sync.sync :as sync]@@ -53,6 +57,61 @@"" [] ; blank -> [], so :key-columns stays schema-validnil [])))+(deftest ^:parallel inline-value-string-test+ (testing "inlined string literals escape the backslash before the quote"+ ;; ClickHouse treats `\` as an escape character inside a string literal, so doubling `'` alone (the default+ ;; `[:sql String]` behaviour) lets a value like `a\'` close the literal early and run the rest as SQL.+ (are [s expected] (= expected (sql.qp/inline-value :clickhouse s))+ "Tito's Tacos" "'Tito\\'s Tacos'"+ "back\\slash" "'back\\\\slash'"+ "' OR 1 = 1 --" "'\\' OR 1 = 1 --'"+ "a\\' OR 1 = 1 --" "'a\\\\\\' OR 1 = 1 --'")))++(def ^:private breakout-payload+ "A value that closes a ClickHouse string literal early unless its backslash is escaped: `a\\' or 1=1 -- `."+ "a\\' or 1=1 -- ")++(def ^:private escaped-breakout-payload+ "[[breakout-payload]] correctly escaped: `\\` is doubled, so the quote that follows is a genuinely escaped quote+ and the payload stays inside the literal."+ "a\\\\\\' or 1=1 -- ")++;;; `contains` / `starts-with` / `ends-with` are an additional carrier for the escaping defect above, and a+;;; very common one. On the generic SQL path they compile to `LIKE <pattern>`, and `sql.qp/generate-pattern` runs+;;; `escape-like-pattern` on the value first -- which doubles `\` and so happens to neutralise this payload shape.+;;; ClickHouse overrides all three to its native scalar functions instead, so `generate-pattern` never runs and the+;;; value reaches ordinary function-argument position unescaped. Only [[sql.qp/inline-value]] stands between it and+;;; the SQL text.+(deftest string-filter-inline-escaping-test+ ;; no ClickHouse server needed -- this only compiles the query -- but the QP pipeline reads the app DB+ (mt/initialize-if-needed! :db)+ (testing "a string filter value cannot break out of the literal when compiled with inline parameters"+ (let [mp (lib.tu/merged-mock-metadata-provider+ meta/metadata-provider+ {:database {:engine :clickhouse+ :dbms-version {:version "24.4" :semantic-version {:major 24 :minor 4}}}})+ venues (lib.metadata/table mp (meta/id :venues))+ name-col (lib.metadata/field mp (meta/id :venues :name))+ compile! (fn [filter-clause]+ (:query (qp.compile/compile-with-inline-parameters+ (-> (lib/query mp venues)+ (lib/filter filter-clause)))))]+ (doseq [[msg filter-clause expected]+ [["contains" (lib/contains name-col breakout-payload)+ (format "`positionUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]+ ["starts-with" (lib/starts-with name-col breakout-payload)+ (format "`startsWithUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]+ ["ends-with" (lib/ends-with name-col breakout-payload)+ (format "`endsWithUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]+ ["case-insensitive contains" (lib/ignore-case (lib/contains name-col breakout-payload))+ (format "`positionCaseInsensitiveUTF8`(`PUBLIC`.`VENUES`.`NAME`, '%s')" escaped-breakout-payload)]+ ["case-insensitive starts-with" (lib/ignore-case (lib/starts-with name-col breakout-payload))+ (format "`lowerUTF8`('%s')" escaped-breakout-payload)]+ ["case-insensitive ends-with" (lib/ignore-case (lib/ends-with name-col breakout-payload))+ (format "`lowerUTF8`('%s')" escaped-breakout-payload)]]]+ (testing msg+ (is (str/includes? (compile! filter-clause) expected)))))))+(deftest ^:parallel clickhouse-version(mt/test-driver :clickhouse(t2.with-temp/with-temp@@ -622,4 +681,7 @@"\"x\"; SELECT sleep(10); --\"" "SET ROLE \"x\"\"; SELECT sleep(10); --\"";; a trailing backslash is escaped so it cannot close the quoted identifier"foo\\" "SET ROLE \"foo\\\\\""- "a\\\"b" "SET ROLE \"a\\\\\"\"b\""))+ "a\\\"b" "SET ROLE \"a\\\\\"\"b\""+ ;; a lone double-quote is a one-character role name, not an already-quoted empty one -- it must still come+ ;; out as a terminated identifier+ "\"" "SET ROLE \"\"\"\""))diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex d76659d7ea8..4cb82f8a33e 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -778,6 +778,15 @@[driver t](sql.qp/->honeysql driver (t/offset-date-time t)))+;;; Snowflake treats `\` inside a string literal as an escape introducer -- `\'` is an escaped single quote and `\\`+;;; an escaped backslash (https://docs.snowflake.com/en/sql-reference/data-types-text). The generic `[:sql String]`+;;; implementation only doubles the single quotes, which leaves a value containing `\'` (or ending in `\`) free to+;;; terminate the literal one quote early. Snowflake accepts both escape forms, so use `:ansi+backslashes`: doubling+;;; the backslash means it can never escape our closing quote.+(defmethod sql.qp/inline-value [:snowflake String]+ [_driver ^String s]+ (sql.u/quote-literal s :ansi+backslashes))+(defmethod driver/table-rows-seq :snowflake[driver database table](driver-api/with-metadata-provider (u/the-id database)@@ -1171,7 +1180,11 @@255)(defn get-string-filter-arg- "Generate the argument to match in the string filters. It's based on sql.qp/generate-pattern."+ "Generate the argument to match in the string filters. It's based on sql.qp/generate-pattern.++ Unlike `sql.qp/generate-pattern` this does no escaping: these filters compile to Snowflake's native scalar+ functions rather than `LIKE`, so there is no pattern to escape, and the value has to stay verbatim to bind+ correctly as a `?` parameter. Escaping the value for an inline compile is [[sql.qp/inline-value]]'s job."[driver[type opts val :as arg]{:keys [case-sensitive] :or {case-sensitive true} :as _options}]diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 8ec8bbaa310..0ae15fa74ed 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -29,6 +29,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.notebook-helpers :as notebook-helpers]+ [metabase.query-processor.compile :as qp.compile]^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp][metabase.secrets.core :as secret]@@ -1733,6 +1734,62 @@(is (str/includes? (-> result :data :native_form :query) exp-filter))(is (= exp-rows (mt/rows result)))))))))+(def ^:private breakout-payload+ "A value that closes a Snowflake string literal early unless its backslash is escaped: `a\\' or 1=1 -- `.+ Kept all-lowercase so the case-insensitive filters, which lower-case the value, expect the same literal."+ "a\\' or 1=1 -- ")++(def ^:private escaped-breakout-payload+ "[[breakout-payload]] correctly escaped: the `\\` is doubled so it cannot escape anything, and the `'` is doubled,+ so the payload stays inside the literal."+ "a\\\\'' or 1=1 -- ")++(deftest ^:parallel inline-value-string-test+ (testing "inlined string literals escape the backslash as well as the quote"+ ;; Snowflake treats `\` as an escape character inside a string literal, so doubling `'` alone (the default+ ;; `[:sql String]` behaviour) lets a value like `a\'` close the literal early and run the rest as SQL.+ (are [s expected] (= expected (sql.qp/inline-value :snowflake s))+ "Tito's Tacos" "'Tito''s Tacos'" ; 'Tito''s Tacos'+ "'" "''''" ; ''''+ "back\\slash" "'back\\\\slash'" ; 'back\\slash'+ "trailing\\" "'trailing\\\\'" ; 'trailing\\'+ breakout-payload (str \' escaped-breakout-payload \'))))++;;; `contains` / `starts-with` / `ends-with` are an additional carrier for the escaping defect above, and a+;;; very common one. On the generic SQL path they compile to `LIKE <pattern>`, and `sql.qp/generate-pattern` runs+;;; `escape-like-pattern` on the value first -- which doubles `\` and so happens to neutralise this payload shape.+;;; Snowflake overrides all three to its native scalar functions instead, so `generate-pattern` never runs and the+;;; value reaches ordinary function-argument position unescaped. Only [[sql.qp/inline-value]] stands between it and+;;; the SQL text.+(deftest string-filter-inline-escaping-test+ ;; no Snowflake warehouse needed -- this only compiles the query -- but the QP pipeline reads the app DB+ (mt/initialize-if-needed! :db)+ (testing "a string filter value cannot break out of the literal when compiled with inline parameters"+ (let [mp (lib.tu/merged-mock-metadata-provider+ meta/metadata-provider+ {:database {:engine :snowflake, :details {:db "test-data"}}})+ venues (lib.metadata/table mp (meta/id :venues))+ name-col (lib.metadata/field mp (meta/id :venues :name))+ compile! (fn [filter-clause]+ (:query (qp.compile/compile-with-inline-parameters+ (-> (lib/query mp venues)+ (lib/filter filter-clause)))))]+ (doseq [[msg filter-clause expected]+ [["contains" (lib/contains name-col breakout-payload)+ (format "CONTAINS(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]+ ["starts-with" (lib/starts-with name-col breakout-payload)+ (format "STARTSWITH(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]+ ["ends-with" (lib/ends-with name-col breakout-payload)+ (format "ENDSWITH(\"PUBLIC\".\"VENUES\".\"NAME\", '%s')" escaped-breakout-payload)]+ ["case-insensitive contains" (lib/ignore-case (lib/contains name-col breakout-payload))+ (format "CONTAINS(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]+ ["case-insensitive starts-with" (lib/ignore-case (lib/starts-with name-col breakout-payload))+ (format "STARTSWITH(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]+ ["case-insensitive ends-with" (lib/ignore-case (lib/ends-with name-col breakout-payload))+ (format "ENDSWITH(LOWER(\"PUBLIC\".\"VENUES\".\"NAME\"), '%s')" escaped-breakout-payload)]]]+ (testing msg+ (is (str/includes? (compile! filter-clause) expected)))))))+(deftest snowflake-collate-comparison-test(mt/test-driver :snowflake(let [mp (mt/metadata-provider)diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex af1f4bb6357..f5be6ccc7e3 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -887,6 +887,21 @@"UTC"offset)))+(defn- utf8-string-literal+ "A MySQL string literal for `s`, written as a character set introducer plus a hex literal so its value does not+ depend on the session's `NO_BACKSLASH_ESCAPES` setting. This has to be a *literal* rather than+ `CONVERT(UNHEX(...) USING utf8mb4)`: a literal keeps coercibility `COERCIBLE` and so takes on the collation of+ whatever it is compared against, while `CONVERT` is `IMPLICIT` and makes every comparison against a column whose+ collation differs from the connection's fail with `Illegal mix of collations`."+ [^String s]+ (format "_utf8mb4 X'%s'" (codecs/bytes->hex (.getBytes s "UTF-8"))))++(defmethod sql.qp/inline-value [:mysql String]+ [_driver ^String s]+ ;; MySQL's interpretation of backslashes in quoted literals depends on `NO_BACKSLASH_ESCAPES`. Use a hex literal so+ ;; both the safety and the value of the resulting string are independent of the session's SQL mode.+ (utf8-string-literal s))+(defmethod sql.qp/inline-value [:mysql OffsetTime][_ t];; MySQL doesn't support timezone offsets in literals so pass in a local time literal wrapped in a call to convert@@ -1360,7 +1375,9 @@(defn- utf8-string-expr"A MySQL expression evaluating to the string `s`, hex-encoded so there is nothing to escape and SQL injection isimpossible. Index names are unvalidated free-form user input and `sql.u/escape-sql` is explicitly not safe for that- (a backslash defeats its quote-doubling, and its escaping is session-dependent), so we use the hex pattern instead."+ (a backslash defeats its quote-doubling, and its escaping is session-dependent), so we use the hex pattern instead.+ Use [[utf8-string-literal]] instead anywhere the result is compared against a column -- this one is `IMPLICIT` and+ would clash with the column's collation."[^String s](format "CONVERT(UNHEX('%s') USING utf8mb4)" (codecs/bytes->hex (.getBytes s "UTF-8"))))diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.cljindex 1c6b4c406e5..afdee820d11 100644--- a/src/metabase/driver/sql/util.clj+++ b/src/metabase/driver/sql/util.clj@@ -104,13 +104,40 @@:else(recur (conj already-seen alias) (conj acc [col alias]) more)))))+(defn- escape-quoted+ "Shared implementation of [[escape-sql]] and [[escape-identifier]]. Escapes `quote-char` -- and, for the styles that+ call for it, the backslash -- so `s` can be spliced between a pair of `quote-char`s without terminating early."+ ^String [^String s ^Character quote-char escape-style]+ (let [q (str quote-char)+ doubled (str quote-char quote-char)]+ (case escape-style+ :ansi (str/replace s q doubled)+ :backslashes (-> s+ (str/replace "\\" "\\\\")+ (str/replace q (str "\\" q)))+ :ansi+backslashes (-> s+ (str/replace "\\" "\\\\")+ (str/replace q doubled)))))+;;; TODO (Cam 2026-04-27) -- rename this to `escape-single-quotes` to make it clearer what we're escaping(defn escape-sql- "Escape single quotes in a SQL string. `escape-style` is either `:ansi` (escape a single quote with two single quotes)- or `:backslashes` (escape a single quote with a backslash).+ "Escape single quotes in a SQL string. `escape-style` is one of++ * `:ansi` -- escape a single quote with two single quotes. Correct only on engines that do *not* treat `\\` as an+ escape character inside a string literal (Postgres with `standard_conforming_strings`, H2, SQL Server, Oracle...).++ * `:backslashes` -- escape the backslash, then escape a single quote with a backslash. For engines that only+ understand the backslash form (BigQuery, Presto...).++ * `:ansi+backslashes` -- escape the backslash, then escape a single quote with two single quotes. For engines that+ always accept both forms, such as Snowflake: doubling the backslash means no `\\` can ever escape our closing quote,+ and doubling the quote terminates the literal where we intended. Do not use this on engines whose backslash+ handling is configurable: it is injection-safe in either mode, but does not preserve the value when backslashes+ are treated literally.- (escape-sql \"Tito's Tacos\" :ansi) ; -> \"Tito''s Tacos\"- (escape-sql \"Tito's Tacos\" :backslashes) ; -> \"Tito\\'s Tacos\"+ (escape-sql \"Tito's Tacos\" :ansi) ; -> \"Tito''s Tacos\"+ (escape-sql \"Tito's Tacos\" :backslashes) ; -> \"Tito\\'s Tacos\"+ (escape-sql \"Tito's Tacos\" :ansi+backslashes) ; -> \"Tito''s Tacos\"!!!! VERY IMPORTANT !!!!@@ -120,14 +147,10 @@encode the strings as hex and splice in something along the lines of `utf8_string(hex_decode(<hex-string>))`instead. This is intended only for escaping trusted strings, or for generating the SQL equivalent version of an MBQLquery for debugging purposes or powering the 'convert to SQL' feature."- {:arglists '([s :ansi] [s :backslashes])}+ {:arglists '([s :ansi] [s :backslashes] [s :ansi+backslashes])}^String [^String s escape-style](when s- (case escape-style- :ansi (str/replace s "'" "''")- :backslashes (-> s- (str/replace "\\" "\\\\")- (str/replace "'" "\\'")))))+ (escape-quoted s \' escape-style)))(defn quote-literal"Wrap `s` in single quotes as a SQL string literal, escaping embedded quotes per `escape-style`.@@ -139,7 +162,41 @@^String [^String s escape-style](when s(case escape-style- (:ansi :backslashes) (str \' (escape-sql s escape-style) \'))))+ (:ansi :backslashes :ansi+backslashes) (str \' (escape-sql s escape-style) \'))))++(defn escape-identifier+ "Escape `s` so it can be spliced between a pair of double quotes as a SQL identifier. `escape-style` is the same+ vocabulary as [[escape-sql]], applied to `\"` rather than `'`:++ * `:ansi` -- double an embedded `\"`. Correct on engines that do *not* treat `\\` as an escape character inside a+ quoted identifier.++ * `:ansi+backslashes` -- escape the backslash as well, so a trailing `\\` cannot escape our closing quote. Needed on+ engines that honour backslash escapes inside quoted identifiers, such as ClickHouse.++ `:backslashes` behaves as it does in [[escape-sql]], applied to `\"`; no current caller needs it.++ (escape-identifier \"a\\\"b\" :ansi) ; -> \"a\\\"\\\"b\"+ (escape-identifier \"trailing\\\\\" :ansi+backslashes) ; -> \"trailing\\\\\\\\\"++ Prefer [[quote-name]] when the driver's `sql.qp/quote-style` already escapes correctly -- it goes through HoneySQL+ and knows about backtick dialects. Reach for this only when hand-building SQL text for an engine whose escaping+ HoneySQL does not cover.++ For user input, *ALWAYS* prefer a query parameter or the engine's own `quote_ident()` where one exists."+ {:arglists '([s :ansi] [s :ansi+backslashes])}+ ^String [^String s escape-style]+ (when s+ (escape-quoted s \" escape-style)))++(defn quote-identifier+ "Wrap `s` in double quotes as a SQL identifier, escaping embedded quotes per `escape-style`. See+ [[escape-identifier]] for the escape styles and for when to prefer [[quote-name]].++ (quote-identifier \"a\\\"b\" :ansi) ; -> \"\\\"a\\\"\\\"b\\\"\""+ ^String [^String s escape-style]+ (when s+ (str \" (escape-identifier s escape-style) \")))(defn validate-convert-timezone-args"Validate the arguments of convert-timezone.diff --git a/src/metabase/warehouse_schema/field_values/distinct_batch.clj b/src/metabase/warehouse_schema/field_values/distinct_batch.cljindex ab3080e60d7..24535281d48 100644--- a/src/metabase/warehouse_schema/field_values/distinct_batch.clj+++ b/src/metabase/warehouse_schema/field_values/distinct_batch.clj@@ -3,11 +3,16 @@For each field in the input set, builds a flat per-field arm of the shape- SELECT '<field-name>' AS field_name, <cast-to-text>(col) AS value_out+ SELECT <arm-ordinal> AS field_idx, <cast-to-text>(col) AS value_outFROM <table>GROUP BY <cast-to-text>(col)<driver-correct LIMIT clause>+ Arms are tagged with their ordinal in the batch rather than with the field's name. The name comes from the+ warehouse verbatim, and an inline string literal built from it is a SQL injection sink on any engine whose+ string lexer we don't escape for; an integer ordinal is unforgeable. It also means two fields that+ happen to share a name stay distinct.+Per-arm DISTINCT semantics come from `GROUP BY` rather than `SELECT DISTINCT` — both areequivalent here, but `GROUP BY` composes cleanly with every driver's[[sql.qp/apply-top-level-clause]] `:limit` transform (notably SQL Server's `TOP N` rewrite,@@ -27,6 +32,7 @@chain-filter → params.field-values → field → field-values) would form a cycle if theserequires lived in `metabase.warehouse-schema.models.field-values`."(:require+ [clojure.string :as str][metabase.driver.sql.query-processor :as sql.qp][metabase.query-processor :as qp][metabase.util :as u]@@ -81,15 +87,15 @@(sql.qp/->honeysql driver [::sql.qp/cast-to-text {} (h2x/identifier :field col-name)]))(defn- build-arm- "HoneySQL for one UNION arm."- [driver table field]+ "HoneySQL for one UNION arm. `idx` is the field's position in the batch and becomes the arm's tag."+ [driver table field idx](let [cast-expr (cast-to-text-honeysql driver (:name field))- inner {:select [[[:inline (:name field)] :field_name]+ inner {:select [[[:inline idx] :field_idx][cast-expr :value_out]];; `:from` wraps the identifier expression in an extra vector — `[[expr]]` so HoneySQL;; treats it as a single table expression rather than `[table alias …]`.:from [[(table-honeysql driver table)]]- ;; Only `cast-expr` goes in `:group-by`. The `[:inline (:name field)]` tag is a literal in+ ;; Only `cast-expr` goes in `:group-by`. The `[:inline idx]` tag is a literal in;; the SELECT list and needs no GROUP BY entry.:group-by [cast-expr]}limited (sql.qp/apply-top-level-clause driver :limit inner {:limit field-values/*distinct-limit*})]@@ -104,11 +110,20 @@(defn- build-union"Build the full HoneySQL form for one batch of fields. Single field → no UNION wrapper."[driver table fields]- (let [arms (mapv #(build-arm driver table %) fields)]+ (let [arms (into [] (map-indexed #(build-arm driver table %2 %1)) fields)](if (= 1 (count arms))(first arms){:union-all arms})))+(defn- decode-arm-idx+ "Coerce the `field_idx` tag a driver returned back to a Clojure integer. Drivers disagree on the type they+ hand back for an inline integer literal (`Long` on most, `BigDecimal` on Oracle/Snowflake, …), so normalize+ rather than assume. Returns `nil` for anything unrecognizable."+ [x]+ (cond+ (number? x) (long x)+ (string? x) (try (Long/parseLong (str/trim x)) (catch NumberFormatException _ nil))))+(defn run-distinct-batch"Execute one UNION ALL query covering `fields` from one `table`. The caller is responsible forensuring `(count fields)` ≤ [[*batch-size*]] (the per-query arm cap) and that all `fields`@@ -123,19 +138,23 @@[table fields](let [db-id (:db_id table)driver (:engine (t2/select-one :model/Database :id db-id))+ fields (vec fields)hsql (build-union driver table fields)[sql & params] (sql.qp/format-honeysql driver hsql)result (qp/process-query{:database db-id:type :native:native {:query sql, :params (vec params)}})- rows (-> result :data :rows)- by-name (into {} (map (juxt :name identity)) fields)]- (reduce (fn [acc [field-name value-str]]- (let [field (get by-name field-name)- v (decode-value (:base_type field) value-str)]- (-> acc- (update-in [(:id field) :values] (fnil conj []) v)- (update-in [(:id field) :raw-count] (fnil inc 0)))))+ rows (-> result :data :rows)]+ (reduce (fn [acc [arm-idx value-str]]+ ;; Arms are tagged by ordinal rather than by name: the name is warehouse-controlled and has no+ ;; business being in the query text as a literal, and ordinals also keep two+ ;; identically-named fields in the same batch from collapsing into one.+ (if-let [field (get fields (decode-arm-idx arm-idx))]+ (let [v (decode-value (:base_type field) value-str)]+ (-> acc+ (update-in [(:id field) :values] (fnil conj []) v)+ (update-in [(:id field) :raw-count] (fnil inc 0))))+ acc))(into {} (map (fn [f] [(:id f) {:values [] :raw-count 0}])) fields)rows)))diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex 775c78f8faf..bb860aa70d0 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -58,6 +58,23 @@(binding [sync-util/*log-exceptions-and-continue?* false](mt/with-test-user :rasta (thunk)))))+(deftest ^:parallel inline-value-string-test+ (testing "inlined strings use a representation that is independent of the session's backslash-escaping mode"+ (let [expected (fn [^String s]+ (format "_utf8mb4 X'%s'" (codecs/bytes->hex (.getBytes s "UTF-8"))))]+ (are [s] (= (expected s) (sql.qp/inline-value :mysql s))+ "Tito's Tacos"+ "back\\slash"+ "a\\' OR 1 = 1 --"+ ""+ "\u00e9\ud83c\udf63")))+ (testing "compiling a query with inline parameters"+ (binding [driver/*compile-with-inline-parameters* true]+ (is (= ["SELECT * FROM `venues` WHERE `venues`.`name` = _utf8mb4 X'615c27204f522031203d2031202d2d'"]+ (sql.qp/format-honeysql :mysql {:select [:*]+ :from [[:venues]]+ :where [:= :venues/name "a\\' OR 1 = 1 --"]}))))))+(deftest all-zero-dates-test(mt/test-driver :mysql(testing (str "MySQL allows 0000-00-00 dates, but JDBC does not; make sure that MySQL is converting them to NULL "diff --git a/test/metabase/driver/sql/util_test.clj b/test/metabase/driver/sql/util_test.cljindex b2c57782a35..a66337842f2 100644--- a/test/metabase/driver/sql/util_test.clj+++ b/test/metabase/driver/sql/util_test.clj@@ -80,7 +80,16 @@"\\\\\\\\' OR 1 = 1 --" "\\\\\\\\\\\\\\\\\\' OR 1 = 1 --""\\\\' OR 1 = 1 --" "\\\\\\\\\\' OR 1 = 1 --""\\' OR 1 = 1 --" "\\\\\\' OR 1 = 1 --"- "' OR 1 = 1 --" "\\' OR 1 = 1 --"}}+ "' OR 1 = 1 --" "\\' OR 1 = 1 --"}++ ;; doubles the backslash *and* the quote, so the literal terminates where we intended whether or not+ ;; the engine treats `\` as an escape character+ :ansi+backslashes+ {"Tito's Tacos" "Tito''s Tacos"+ "\\\\\\\\' OR 1 = 1 --" "\\\\\\\\\\\\\\\\'' OR 1 = 1 --"+ "\\\\' OR 1 = 1 --" "\\\\\\\\'' OR 1 = 1 --"+ "\\' OR 1 = 1 --" "\\\\'' OR 1 = 1 --"+ "' OR 1 = 1 --" "'' OR 1 = 1 --"}}[s expected] s->expected](testing escape-strategy@@ -95,9 +104,49 @@(testing ":backslashes escape style"(is (= "'Tito\\'s Tacos'"(sql.u/quote-literal "Tito's Tacos" :backslashes))))+ (testing ":ansi+backslashes escape style"+ (is (= "'Tito''s Tacos'"+ (sql.u/quote-literal "Tito's Tacos" :ansi+backslashes)))+ (is (= "'a\\\\'' OR 1 = 1 --'"+ (sql.u/quote-literal "a\\' OR 1 = 1 --" :ansi+backslashes))))(testing "empty string"(is (= "''" (sql.u/quote-literal "" :ansi)))))+(deftest ^:parallel escape-identifier-test+ (doseq [[escape-strategy s->expected]+ {:ansi+ {"role" "role"+ "a\"b" "a\"\"b"+ "trailing\\" "trailing\\"+ "a\\\"b" "a\\\"\"b"}++ ;; doubles the backslash *and* the quote, so the identifier terminates where we intended whether or not+ ;; the engine treats `\` as an escape character. This is what ClickHouse needs.+ :ansi+backslashes+ {"role" "role"+ "a\"b" "a\"\"b"+ "trailing\\" "trailing\\\\"+ "a\\\"b" "a\\\\\"\"b"}}++ [s expected] s->expected]+ (testing escape-strategy+ (testing (pr-str s)+ (is (= expected+ (sql.u/escape-identifier s escape-strategy)))))))++(deftest ^:parallel quote-identifier-test+ (testing "wraps in double quotes and doubles interior quotes"+ (is (= "\"role\"" (sql.u/quote-identifier "role" :ansi)))+ (is (= "\"a\"\"b\"" (sql.u/quote-identifier "a\"b" :ansi))))+ (testing ":ansi+backslashes also escapes backslashes"+ (testing "a trailing backslash cannot escape our closing quote"+ (is (= "\"trailing\\\\\"" (sql.u/quote-identifier "trailing\\" :ansi+backslashes))))+ (is (= "\"a\\\\\"\"b\"" (sql.u/quote-identifier "a\\\"b" :ansi+backslashes))))+ (testing "empty string"+ (is (= "\"\"" (sql.u/quote-identifier "" :ansi))))+ (testing "nil"+ (is (nil? (sql.u/quote-identifier nil :ansi)))))+;;; Ok to hardcode driver names in the tests below because they're for general util functions and not something that;;; needs to be run against all supported driversdiff --git a/test/metabase/warehouse_schema/models/field_values_test.clj b/test/metabase/warehouse_schema/models/field_values_test.cljindex eb7f4613e5c..cfa88c11d89 100644--- a/test/metabase/warehouse_schema/models/field_values_test.clj+++ b/test/metabase/warehouse_schema/models/field_values_test.clj@@ -3,10 +3,13 @@the [[metabase.warehouse-schema.models.field-values]] namespace."(:require[clojure.java.jdbc :as jdbc]+ [clojure.string :as str][clojure.test :refer :all][java-time.api :as t][metabase.driver :as driver][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.query-processor :as qp][metabase.sync.core :as sync][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -632,6 +635,57 @@(testing "Unknown base-type → string passthrough"(is (= "anything" (#'distinct-batch/decode-value :type/SomeMadeUpType "anything")))))+;;; Column names are harvested verbatim from the warehouse by `describe-table` / `describe-fields`, so they are+;;; attacker-controlled for anyone who can create a column in a synced schema. They must never reach the generated+;;; SQL as a string literal -- only as a quoted identifier.+(def ^:private sql-injection-field-name+ "a\\' AS `field_name`, (SELECT @@version) AS `value_out` FROM mysql.db LIMIT 1) AS `_arm` -- ")++(deftest ^:parallel build-union-tags-arms-by-ordinal-test+ (testing "arms are tagged with their ordinal, not with the field name"+ (let [fields [{:name "state" :base_type :type/Text}+ {:name "source" :base_type :type/Text}]]+ (doseq [[driver q] {:h2 "\"", :postgres "\"", :mysql "`"}]+ (testing driver+ (let [sql (first (sql.qp/format-honeysql driver (#'distinct-batch/build-union driver {:name "t"} fields)))]+ (is (str/includes? sql (str "0 AS " q "field_idx" q)))+ (is (str/includes? sql (str "1 AS " q "field_idx" q)))+ (is (not (str/includes? sql "'"))+ (str "no SQL string literal is generated at all. Got: " sql))))))))++(deftest ^:parallel build-union-field-name-cannot-break-out-of-sql-test+ (testing "a warehouse column name crafted to break out of a string literal stays inert"+ ;; On MySQL `\'` is a second way to write a quote inside a literal, so the old `[:inline (:name field)]` tag+ ;; -- escaped only by doubling `'` -- closed early and the rest of the column name ran as SQL. Post-fix+ ;; there is no tag literal at all, and the name reaches the query only as a quoted identifier (which is+ ;; where the column genuinely is), with the identifier quote character doubled.+ (let [fields [{:name sql-injection-field-name :base_type :type/Text}]+ sql (first (sql.qp/format-honeysql :mysql (#'distinct-batch/build-union :mysql {:name "t"} fields)))+ quoted-name (str "`" (str/replace sql-injection-field-name "`" "``") "`")]+ (is (= (str "SELECT * FROM ("+ "SELECT 0 AS `field_idx`, CAST(" quoted-name " AS char) AS `value_out` "+ "FROM `t` "+ "GROUP BY CAST(" quoted-name " AS char) "+ "LIMIT 1000) AS `_arm`")+ sql)))))++(deftest run-distinct-batch-demuxes-by-arm-ordinal-test+ (testing "rows are mapped back to fields by arm ordinal, not by name"+ ;; `idx_unique_field` only makes `name` unique per (table, parent) pair, so a table can legitimately hold two+ ;; fields called `dupe` as long as one of them is nested. Keying the demux by name collapsed both arms onto+ ;; whichever field won the `by-name` lookup; the ordinal keeps them apart.+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table {table-id :id :as table} {:db_id db-id :name "t"}+ :model/Field {parent-id :id} {:table_id table-id :name "json_col" :base_type :type/JSON}+ :model/Field {f1-id :id} {:table_id table-id :name "dupe" :base_type :type/Text}+ :model/Field {f2-id :id} {:table_id table-id :name "dupe" :base_type :type/Integer+ :parent_id parent-id}]+ (let [fields [(t2/select-one :model/Field :id f1-id) (t2/select-one :model/Field :id f2-id)]]+ (with-redefs [qp/process-query (fn [_query] {:data {:rows [[0 "a"] [0 "b"] [1 "42"]]}})]+ (is (= {f1-id {:values ["a" "b"] :raw-count 2}+ f2-id {:values [42] :raw-count 1}}+ (distinct-batch/run-distinct-batch table fields))))))))+(deftest ^:mb/driver-tests run-distinct-batch-integration-test(testing "run-distinct-batch returns correct distinct values for each field"(mt/test-drivers (sql-test-drivers)