The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Bind inline values in app-DB queries with [:auto/param ...] (#82373)
bryan4a3d32ae0ae8
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 80b9f733ad8..71bcd58f43f 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -74,6 +74,10 @@:metabase/prefer-with-dynamic-fn-redefs {:level :warning}:metabase/t2-query-namespace {:level :warning}:metabase/table-or-field-query {:level :warning}+ ;; App-DB query hygiene: a value reaching a value slot unmarked, and a query written as+ ;; `:column value` pairs rather than a map. Off by default and enabled per namespace under+ ;; `:config-in-ns`; flip it on globally once every `db.clj` has been converted.+ :metabase/unsafe-app-db-query {:level :off}:metabase/toucan-model-ns {:level :warning}:metabase/require-shape-checker {:level :warning}:metabase/tests-must-live-in-known-modules {:level :warning}@@ -924,6 +928,11 @@app-db-namespaces{:linters {:metabase/t2-query-namespace {:level :off}}}+ ;; Namespaces whose values are marked. Adding one here is what keeps it that way; see+ ;; `metabase.app-db.value-guard`.+ metabase.content-translation.db+ {:linters {:metabase/unsafe-app-db-query {:level :warning}}}+;; .cljc tests run in both Clojure and ClojureScript — `with-dynamic-fn-redefs` is JVM-only,;; so leave these alone.metabase.lib.aggregation-testdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex dcb6c2b694f..6fe6d144cf5 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -62,6 +62,7 @@:metabase/prefer-with-dynamic-fn-redefs 4:metabase/t2-query-namespace 2:metabase/table-or-field-query 1+ :metabase/unsafe-app-db-query 1:missing-docstring 1:private-call 1:redundant-ignore 1diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.cljindex 660d2d75c37..e3f3a4b30f6 100644--- a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj@@ -29,6 +29,100 @@[filename](boolean (and filename (re-find #"(?:^|/)test/" filename))))+(def ^:private value-operators+ "Operators whose second argument is a value rather than a column."+ '#{= not= < > <= >= like not-like ilike not-ilike in not-in between})++(defn- marked?+ "Whether `node` is an `[:auto/param v]` marker."+ [node]+ (and (hooks/vector-node? node)+ (= :auto/param (some-> (first (:children node)) hooks/sexpr))))++(defn- value-nodes+ "The nodes sitting in a value slot of `node`, following the clause shapes a query map uses."+ [node]+ (cond+ ;; A query map -- walk its clause values, not its keys.+ (hooks/map-node? node)+ (mapcat value-nodes (take-nth 2 (rest (:children node))))++ (hooks/vector-node? node)+ (let [[head & args] (:children node)+ op (some-> head hooks/sexpr)]+ (cond+ (contains? #{:and :or :not} op)+ (mapcat value-nodes args)++ ;; `[:= col v]`, `[:in col vs]`, `[:between col lo hi]` -- the first argument is the+ ;; column and the rest are values, whatever the arity.+ (and (keyword? op) (contains? value-operators (symbol (name op))))+ (rest args)++ :else+ (mapcat value-nodes (:children node))))++ :else nil))++(def ^:private write-fns+ "Calls whose values are written rather than filtered on. A value here is not a where-clause value+ an attacker can turn into SQL structure, and a `define-before-insert` hook may read it before the+ query compiles, so a marker would break it rather than protect anything."+ '#{insert! insert-returning-instance! insert-returning-instances! insert-returning-pk!+ insert-returning-pks! save!})++(defn- read-call?+ "Whether `node` is a query call whose values are filtered on rather than written."+ [node]+ (let [f (some-> (first (:children node)) hooks/sexpr)]+ (not (contains? write-fns (some-> f name symbol)))))++(defn- kv-arg-pair+ "The first `:column value` pair a query call passes after its model, if any.++ `(t2/select :model/X :locale locale)`. Several of these fns take an argument before the model --+ `(t2/select-one-fn :value :model/X :key k)` -- so the pairs do not start at a fixed offset; they+ start after the `:model/...` keyword. A call that does not name a literal model is not checked."+ [args]+ (let [after-model (->> args+ (drop-while #(not (and (hooks/keyword-node? %)+ (= "model" (namespace (hooks/sexpr %))))))+ rest)+ [k v] after-model]+ (when (and k v (hooks/keyword-node? k))+ k)))++(defn- lint-kv-args!+ "Register a finding for a query written as `:column value` pairs.++ Toucan builds those into the where clause itself, so the query's shape is not on the page: there+ is nothing for a reader, this linter, or the `.sql` extraction that follows this project to read.+ The same call takes a map -- `{:where [:= :locale v]}` for a select, `{:locale v}` for an+ `update!` -- which is the statement, near enough to read off."+ [node]+ (when-let [k (kv-arg-pair (rest (:children node)))]+ (hooks/reg-finding!+ (assoc (meta k)+ :message (format "Pass this query a map rather than `%s ...` pairs, so its shape is on the page."+ (hooks/sexpr k))+ :type :metabase/unsafe-app-db-query))))++(defn- lint-unmarked-values!+ "Register a finding for each argument of the enclosing function that reaches a value slot unmarked.++ Only a symbol is reported. A literal cannot carry a request value, and a value built inside the+ function is out of reach of a check that does not follow it across a call."+ [node]+ (doseq [value (mapcat value-nodes (rest (:children node)))+ :when (and (hooks/token-node? value)+ (symbol? (hooks/sexpr value))+ (not (marked? value)))]+ (hooks/reg-finding!+ (assoc (meta value)+ :message (format "`%s` reaches a SQL value slot unmarked. Write it as [:auto/param %s] so it is bound as a parameter."+ (hooks/sexpr value) (hooks/sexpr value))+ :type :metabase/unsafe-app-db-query))))+(defn lint-query-call"Register a `:metabase/t2-query-namespace` finding when a Toucan 2 query call appears outside a `<module>.db`namespace."@@ -43,4 +137,10 @@(hooks/sexpr fn-node)):type :metabase/t2-query-namespace))))((requiring-resolve 'hooks.metabase.warehouse-schema-overlay.table-or-field-query/lint-read) input)+ (when (and ns+ (db-namespace? (modules/config input) ns)+ (not (test-file? filename))+ (read-call? node))+ (lint-unmarked-values! node)+ (lint-kv-args! node))input)diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.cljindex 99bd57cd346..1010799e58c 100644--- a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj@@ -5,8 +5,15 @@[clojure.test :refer :all][hooks.metabase.toucan.db-ns :as toucan.db-ns]))+(defn- confinement-findings+ "Only the `t2-query-namespace` findings, so these tests are not affected by the query-hygiene+ linter that shares this hook."+ [findings]+ (filter #(= :metabase/t2-query-namespace (:type %)) findings))+(defn- lint-query-call [form ns-sym & [filename modules]]- (binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/t2-query-namespace {:level :warning}}}+ (binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/t2-query-namespace {:level :warning}+ :metabase/unsafe-app-db-query {:level :warning}}}:ignores (atom nil):findings (atom []):namespaces (atom {})}]@@ -25,11 +32,11 @@:message #".*`t2/select-one`.*"}](lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.models.card))))(testing "metabase.<module>.db is allowed"- (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.db))))+ (is (empty? (confinement-findings (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.db)))))(testing "metabase-enterprise.<module>.db is allowed"- (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase-enterprise.sandbox.db))))+ (is (empty? (confinement-findings (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase-enterprise.sandbox.db)))))(testing "metabase.driver.<driver>.db is allowed for driver modules"- (is (empty? (lint-query-call '(t2/select-one :model/Database :id 1) 'metabase.driver.bigquery-cloud-sdk.db))))+ (is (empty? (confinement-findings (lint-query-call '(t2/select-one :model/Database :id 1) 'metabase.driver.bigquery-cloud-sdk.db)))))(testing "a nested db namespace is not a module db namespace"(is (=? [{:type :metabase/t2-query-namespace}](lint-query-call '(t2/query {:select [:*]}) 'metabase.queries.models.db))))@@ -71,3 +78,61 @@(lint-query-call '(t2/query {:select [:*]}) 'metabase.metabot.llm.models.db nil modules))))(testing "enterprise modules resolve through the metabase-enterprise root"(is (empty? (lint-query-call '(t2/select :model/Card) 'metabase-enterprise.sandbox.db nil modules))))))++(deftest ^:parallel unmarked-value-in-a-db-namespace-test+ (testing "a symbol reaching a value slot is flagged"+ (is (=? [{:type :metabase/unsafe-app-db-query+ :message #"`locale` reaches a SQL value slot unmarked.*"}]+ (lint-query-call '(t2/select :model/X {:where [:= :locale locale]}) 'metabase.foo.db))))+ (testing "a marked value is not flagged"+ (is (empty? (lint-query-call '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})+ 'metabase.foo.db))))+ (testing "a literal cannot carry a request value and is not flagged"+ (is (empty? (lint-query-call '(t2/select :model/X {:where [:= :locale "de"]}) 'metabase.foo.db))))+ (testing "a column reference is not a value"+ (is (empty? (lint-query-call '(t2/select :model/X {:where [:= :a.id :b.id]}) 'metabase.foo.db))))+ (testing "values nested under a boolean connective are reached"+ (is (=? [{:message #"`b`.*"}]+ (lint-query-call '(t2/select :model/X {:where [:and [:= :x [:auto/param a]] [:= :y b]]})+ 'metabase.foo.db))))+ (testing "only db namespaces are linted for unmarked values"+ (is (empty? (filter #(= :metabase/unsafe-app-db-query (:type %))+ (lint-query-call '(t2/select :model/X {:where [:= :locale locale]})+ 'metabase.foo.models.thing)))))+ (testing "a test source tree is exempt"+ (is (empty? (lint-query-call '(t2/select :model/X {:where [:= :locale locale]})+ 'metabase.foo.db "test/metabase/foo/db_test.clj")))))++(deftest ^:parallel kv-arg-style-test+ (testing "a query written as :column value pairs is flagged whatever the values are"+ (are [form] (=? [{:type :metabase/unsafe-app-db-query, :message #"Pass this query a map.*"}]+ (lint-query-call form 'metabase.foo.db))+ '(t2/select :model/X :locale locale)+ '(t2/select :model/X :archived false)+ '(t2/select :model/X :locale [:auto/param locale])+ '(t2/select-one :model/X :id (long id))))+ (testing "one finding per call, not one per pair"+ (is (= 1 (count (lint-query-call '(t2/select :model/X :locale locale :msgid msgid)+ 'metabase.foo.db)))))+ (testing "a fn with an argument before the model is still recognised"+ (is (=? [{:message #"Pass this query a map.*"}]+ (lint-query-call '(t2/select-one-fn :value :model/Setting :key k) 'metabase.foo.db))))+ (testing "a map query is not flagged"+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))+ '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})+ '(t2/update! :model/X {:locale [:auto/param locale]} {:msgstr "x"}))))++(deftest ^:parallel write-calls-are-not-linted-for-values-test+ (testing "an insert's values are written, not filtered on, so they are not flagged"+ (are [form] (empty? (filter #(= :metabase/unsafe-app-db-query (:type %))+ (lint-query-call form 'metabase.foo.db)))+ '(t2/insert! :model/X :key k :value v)+ '(t2/insert-returning-instances! :model/X :key k :value v)))+ (testing "a select's values are still flagged"+ (is (=? [{:type :metabase/unsafe-app-db-query}]+ (lint-query-call '(t2/select :model/X :key k) 'metabase.foo.db)))))++(deftest ^:parallel operator-arity-test+ (testing "a value operator with an unexpected arity still has its args examined"+ (is (seq (lint-query-call '(t2/select :model/X {:where [:= :a b c]}) 'metabase.foo.db)))+ (is (seq (lint-query-call '(t2/select :model/X {:where [:between :a lo hi]}) 'metabase.foo.db)))))diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex 24667337c76..80b17e93f3a 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -24,6 +24,7 @@[metabase.app-db.query-cancelation][metabase.app-db.setup :as mdb.setup][metabase.app-db.spec :as mdb.spec]+ [metabase.app-db.value-guard][metabase.config.core :as config][potemkin :as p]))@@ -32,7 +33,8 @@(comment metabase.app-db.format/keep-memetabase.app-db.honeysql-guard/keep-memetabase.app-db.query/keep-me- metabase.app-db.query-cancelation/keep-me)+ metabase.app-db.query-cancelation/keep-me+ metabase.app-db.value-guard/keep-me)(p/import-vars[mdb.connectiondiff --git a/src/metabase/app_db/value_guard.clj b/src/metabase/app_db/value_guard.cljnew file mode 100644index 00000000000..ac91cb5df2e--- /dev/null+++ b/src/metabase/app_db/value_guard.clj@@ -0,0 +1,175 @@+(ns metabase.app-db.value-guard+ "Binds a value written inline in an app-DB query as a SQL parameter.++ HoneySQL resolves a map in a value slot toward SQL *structure* rather than data, so a value that+ arrives from a request can reach the database as syntax. Marking it keeps it data:++ (t2/select :model/ContentTranslation+ {:where [:= :locale [:auto/param locale]]})+ ;; => [\"SELECT ... WHERE locale = ?\" \"de\"]++ HoneySQL binds a value through `[:param :k]` against a separate params map. `auto-param` rewrites+ the inline marker into that pair at the compile step, so a caller writes the value where it+ belongs and never keeps the two in sync.++ A marker is not what stops a hostile non-scalar. `honeysql-guard` runs `:before` this `:around`,+ so it sees the payload still inline and rejects a `{:raw ...}` or a bare subquery there whether or+ not it was marked. What the marker adds is that an ordinary value -- a string, a locale, a token --+ is bound rather than left for HoneySQL to interpret.++ Put a marker in a value slot. Written anywhere else it is rewritten into a `[:param :k]` that+ HoneySQL formats as an identifier rather than binding, so the value is dropped and the generated+ key lands in the statement:++ {:from [[:auto/param \"core_user\"]]} ;; => [\"SELECT * FROM param AS p33yf8xpiqaxw\"]++ Nothing here catches that: HoneySQL gives no signal for a param it did not consume, and which+ positions bind is a decision it makes per operator, so it cannot be inferred from the query."+ (:require+ [clojure.walk :as walk]+ [methodical.core :as methodical]+ [toucan2.honeysql2 :as t2.honeysql]+ [toucan2.pipeline :as t2.pipeline])+ (:import+ (clojure.lang MapEntry)+ (java.util.concurrent ThreadLocalRandom)))++(set! *warn-on-reflection* true)++(defn- marker-form?+ "Whether `x` is marker-shaped -- headed by `:auto/param`.++ Broader than what [[auto-param]] will lift, so that a marker written some other way is rejected+ rather than left to compile. A `MapEntry` is excluded: it is a two-element vector holding its own+ key, so `{:auto/param v}` (a column of that name) would otherwise look like a marker."+ [x]+ (and (sequential? x)+ (not (instance? MapEntry x))+ (= :auto/param (first x))))++(defn- well-formed-marker?+ "Whether `x` is a marker this can lift.++ Two shapes reach here. `[:auto/param v]` is what a caller writes in a query map. Toucan builds a+ kv-arg -- `(t2/select :model/X :locale [:auto/param v])` -- into `[:auto/param :locale v]`, taking+ the marker keyword for the operator and folding the column in, so that arity is a marked kv-arg+ rather than a mistake."+ [x]+ (and (marker-form? x)+ (vector? x)+ (contains? #{2 3} (count x))))++(defn- operator-form?+ "Whether `v` looks like a HoneySQL operator form -- `[:in [...]]`, `[:not-between lo hi]`.++ Any keyword heads an operator as far as Toucan is concerned, so this asks whether the payload is+ keyword-headed rather than checking against a list of known operators. Listing them would let an+ unlisted one through to be bound as a value, which changes the comparison rather than failing."+ [v]+ (and (sequential? v)+ (keyword? (first v))+ (not= :auto/param (first v))))++(defn- kv-arg-marker?+ "Whether `x` is the `[:auto/param column v]` form Toucan builds from a marked kv-arg."+ [x]+ (and (well-formed-marker? x)+ (= 3 (count x))))++(defn- check-well-formed!+ "A marker-shaped form that is not `[:auto/param v]` is always a mistake. HoneySQL does not know the+ marker, so whatever is left compiles into a call to a function named PARAM and fails at the+ database with nothing pointing back here."+ [x]+ (when-not (well-formed-marker? x)+ (throw (ex-info (str "Malformed [:auto/param ...] marker: " (pr-str x)+ ". Write it as a two-element vector, [:auto/param value].")+ {:type ::malformed-marker, :form x}))))++(defn- contains-marker?+ "Whether any marker survives in `form`."+ [form]+ (let [found (volatile! false)]+ (walk/postwalk (fn [x] (when (marker-form? x) (vreset! found true)) x) form)+ @found))++(defn- param-key+ "A random key for a lifted value. Random rather than sequential so that a `[:param :k]` arriving+ in request data cannot name a slot this query minted."+ []+ (keyword (str "p" (Long/toUnsignedString (.nextLong (ThreadLocalRandom/current)) 36))))++(defn- auto-param+ "Rewrite `[:auto/param v]` markers in `query` into HoneySQL's `[:param :kN]`, returning+ `[rewritten-query params-map]`.++ Does not descend into a marker's payload: whatever a caller marked is the value, even when that+ value is itself shaped like a marker."+ [query]+ (let [params (volatile! {})+ walked (walk/prewalk+ (fn [x]+ (if-not (marker-form? x)+ x+ ;; Replace the whole form, so the payload is never descended into.+ (do+ (check-well-formed! x)+ (let [kv? (kv-arg-marker? x)+ v (if kv? (nth x 2) (second x))+ _ (when (operator-form? v)+ ;; Binding an operator form would make it the value of a+ ;; comparison rather than the comparison itself, turning+ ;; `IN (?, ?)` into `= ?` against a list. The marker goes inside.+ (throw (ex-info (str "Marked a whole operator form: " (pr-str x)+ ". Put the marker on the value instead, e.g. "+ "[" (first v) " [:auto/param ...]].")+ {:type ::marked-operator-form, :form x})))+ ;; A marked kv-arg has to come back out as a comparison, since Toucan+ ;; folded the column into the marker rather than building one.+ wrap (if kv? #(vector := (second x) %) identity)]+ (when (and (coll? v) (empty? v) (not (map? v)))+ ;; Toucan rewrites `[:in col []]` to `false`, because `IN ()` is invalid+ ;; SQL, and that rewrite runs inside the compile step this wraps -- so a+ ;; lifted empty collection would hide it and leave `IN ()`, which Postgres+ ;; rejects and H2 quietly accepts. Whether the rewrite applies depends on+ ;; the enclosing operator, which is not visible here, so refuse rather+ ;; than guess.+ (throw (ex-info (str "Marked an empty collection: " (pr-str x)+ ". Leave it unmarked -- an empty collection is not a"+ " value that needs binding, and Toucan rewrites an"+ " empty `:in` that it can see.")+ {:type ::marked-empty-collection, :form x})))+ (if (nil? v)+ ;; HoneySQL turns a literal nil in a comparison into `IS NULL`, where a+ ;; bound parameter would get `= ?` and match nothing. Leave it to HoneySQL.+ (wrap v)+ (let [k (param-key)]+ (vswap! params assoc k v)+ (wrap [:param k])))))))+ query)]+ [walked @params]))++(defn- assert-no-marker-survived!+ "A marker must never reach SQL. HoneySQL does not recognise it and compiles the leftover form into+ a call to a function named PARAM, or -- in a slot it formats as an identifier -- into the literal+ identifier `param`, silently discarding the value."+ [query]+ (when (contains-marker? query)+ (throw (ex-info "[:auto/param ...] would reach SQL unlifted. It belongs in a value slot of a query map."+ {:type ::marker-reached-sql, :query query}))))++(methodical/defmethod t2.pipeline/compile :around :default+ [query-type model built-query]+ ;; Toucan re-enters `compile` with the `[sql & args]` vector it produced, so only a map is worth+ ;; walking -- and skipping the rest keeps this off the second pass.+ (if-not (map? built-query)+ (do (assert-no-marker-survived! built-query)+ (next-method query-type model built-query))+ (let [[query params] (auto-param built-query)]+ (assert-no-marker-survived! query)+ (if-not (seq params)+ (next-method query-type model query)+ ;; HoneySQL takes params as a format option rather than a query clause. Merge so that an+ ;; enclosing `*options*` keeps whatever params it already carried.+ (binding [t2.honeysql/*options* (update (t2.honeysql/options) :params merge params)]+ (next-method query-type model query))))))diff --git a/src/metabase/content_translation/db.clj b/src/metabase/content_translation/db.cljindex a2b8e689a8e..3190792af50 100644--- a/src/metabase/content_translation/db.clj+++ b/src/metabase/content_translation/db.clj@@ -8,7 +8,11 @@(mu/defn translations-for-locale"The ContentTranslations for `locale`, ordered by message id."[locale :- :string]- (t2/select :model/ContentTranslation :locale locale {:order-by [:msgid]}))+ ;; `locale` arrives from a request parameter, so it is bound as a SQL parameter rather than+ ;; compiled into the query.+ (t2/select :model/ContentTranslation+ {:where [:= :locale [:auto/param locale]]+ :order-by [:msgid]}))(mu/defn all-translations"Every ContentTranslation, ordered by locale and message id."diff --git a/test/metabase/app_db/empty_in_test.clj b/test/metabase/app_db/empty_in_test.cljnew file mode 100644index 00000000000..71da1ec6d3b--- /dev/null+++ b/test/metabase/app_db/empty_in_test.clj@@ -0,0 +1,37 @@+(ns metabase.app-db.empty-in-test+ "A marked empty collection is refused: whether Toucan's empty-`:in` rewrite applies depends on the+ enclosing operator, which the lift cannot see."+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest marked-empty-collection-is-rejected-test+ (are [form] (thrown-with-msg? clojure.lang.ExceptionInfo #"Marked an empty collection"+ (t2/select :model/ContentTranslation form))+ {:where [:in :id [:auto/param []]]}+ {:where [:= :locale [:auto/param []]]}))++(deftest an-unmarked-empty-in-is-still-rewritten-test+ (testing "leaving it unmarked keeps Toucan's rewrite, which turns IN () into FALSE"+ ;; Match the clause rather than the whole statement: each app DB quotes identifiers its own way.+ (let [[sql] (t2/compile (t2/select :model/ContentTranslation {:where [:in :id []]}))]+ (is (re-find #"(?i)WHERE FALSE" sql))+ (is (not (re-find #"IN \(\)" sql))))))++(deftest a-non-empty-collection-still-binds-test+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "a" :msgstr "b"}+ :model/ContentTranslation _ {:locale "fr" :msgid "c" :msgstr "d"}]+ (is (= ["de" "fr"]+ (sort (map :locale (t2/select :model/ContentTranslation+ {:where [:in :locale [:auto/param ["de" "fr"]]]})))))))++(deftest an-empty-map-is-not-an-empty-collection-test+ (testing "a map is a value rather than a collection of them, so the refusal does not apply"+ ;; `honeysql-guard` rejects a bare map in a value slot before the lift sees it, so assert on+ ;; which guard fires rather than on the query running.+ (let [e (try (t2/select :model/ContentTranslation {:where [:= :locale [:auto/param {}]]})+ nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? e))+ (is (not= :metabase.app-db.value-guard/marked-empty-collection (:type (ex-data e)))))))diff --git a/test/metabase/app_db/kv_test.clj b/test/metabase/app_db/kv_test.cljnew file mode 100644index 00000000000..20a8e051fe9--- /dev/null+++ b/test/metabase/app_db/kv_test.clj@@ -0,0 +1,25 @@+(ns metabase.app-db.kv-test+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest kv-arg-marker-binds-test+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "hello" :msgstr "hallo"}+ :model/ContentTranslation _ {:locale "fr" :msgid "hello" :msgstr "bonjour"}]+ (testing "a marker in a kv-arg binds the value"+ (is (= ["hallo"] (map :msgstr (t2/select :model/ContentTranslation+ :locale [:auto/param "de"])))))+ (testing "a value that looks like SQL matches nothing"+ (is (empty? (t2/select :model/ContentTranslation :locale [:auto/param "de' OR '1'='1"]))))+ (testing "several marked kv-args each bind"+ (is (= ["hallo"] (map :msgstr (t2/select :model/ContentTranslation+ :locale [:auto/param "de"]+ :msgid [:auto/param "hello"])))))+ (testing "marked and unmarked kv-args together"+ (is (= ["hallo"] (map :msgstr (t2/select :model/ContentTranslation+ :locale [:auto/param "de"]+ :msgid "hello")))))+ (testing "other t2 fns take a marked kv-arg"+ (is (t2/exists? :model/ContentTranslation :locale [:auto/param "de"]))+ (is (= 1 (t2/count :model/ContentTranslation :locale [:auto/param "de"]))))))diff --git a/test/metabase/app_db/kvop_test.clj b/test/metabase/app_db/kvop_test.cljnew file mode 100644index 00000000000..7d1d983a15b--- /dev/null+++ b/test/metabase/app_db/kvop_test.clj@@ -0,0 +1,28 @@+(ns metabase.app-db.kvop-test+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest marker-goes-on-the-value-not-the-operator-test+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "a" :msgstr "b"}+ :model/ContentTranslation _ {:locale "fr" :msgid "c" :msgstr "d"}]+ (testing "a marker inside an operator form binds the value"+ (is (= ["de" "fr"]+ (sort (map :locale (t2/select :model/ContentTranslation+ :locale [:in [:auto/param ["de" "fr"]]])))))+ (is (= ["fr"]+ (map :locale (t2/select :model/ContentTranslation+ :locale [:not= [:auto/param "de"]])))))+ (testing "marking the whole operator form throws rather than changing the comparison"+ (are [form] (thrown-with-msg? clojure.lang.ExceptionInfo #"Marked a whole operator form"+ (t2/select :model/ContentTranslation form))+ ;; a query map, not only a kv-arg+ {:where [:= :locale [:auto/param [:in ["de"]]]]}+ {:where [:= :locale [:auto/param [:lower :x]]]})+ (are [v] (thrown-with-msg? clojure.lang.ExceptionInfo #"Marked a whole operator form"+ (t2/select :model/ContentTranslation :locale v))+ [:auto/param [:in ["de" "fr"]]]+ ;; an operator the check does not have to know by name+ [:auto/param [:not-between 1 5]]+ [:auto/param [:regexp "x"]]))))diff --git a/test/metabase/app_db/params_test.clj b/test/metabase/app_db/params_test.cljnew file mode 100644index 00000000000..2dce541fca6--- /dev/null+++ b/test/metabase/app_db/params_test.clj@@ -0,0 +1,63 @@+(ns metabase.app-db.params-test+ "End-to-end: a marked value reaches the database as a parameter."+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(deftest marked-value-is-bound-test+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "hello" :msgstr "hallo"}+ :model/ContentTranslation _ {:locale "fr" :msgid "hello" :msgstr "bonjour"}]+ (testing "the query filters on the bound value"+ (is (= ["hallo"]+ (map :msgstr (t2/select :model/ContentTranslation+ {:where [:= :locale [:auto/param "de"]]})))))+ (testing "a value that looks like SQL is compared as data and matches nothing"+ (is (empty? (t2/select :model/ContentTranslation+ {:where [:= :locale [:auto/param "de' OR '1'='1"]]}))))+ (testing "several markers in one query each bind separately"+ (is (= ["hallo"]+ (map :msgstr (t2/select :model/ContentTranslation+ {:where [:and+ [:= :locale [:auto/param "de"]]+ [:= :msgid [:auto/param "hello"]]]})))))))++(deftest marker-binds-on-the-write-path-test+ (testing "a marked value is bound in an update"+ (mt/with-temp [:model/ContentTranslation {id :id} {:locale "de" :msgid "a" :msgstr "b"}]+ (t2/update! :model/ContentTranslation id {:msgstr [:auto/param "updated"]})+ (is (= "updated" (:msgstr (t2/select-one :model/ContentTranslation :id id))))))+ (testing "and in an insert"+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "a" :msgstr "b"}]+ (t2/insert! :model/ContentTranslation {:locale [:auto/param "fr"] :msgid "x" :msgstr "y"})+ (is (= ["y"] (map :msgstr (t2/select :model/ContentTranslation :locale "fr")))))))++(deftest a-query-without-markers-is-unchanged-test+ (testing "a plain query still runs -- the compile step leaves it alone"+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "a" :msgstr "b"}]+ (is (= ["b"] (map :msgstr (t2/select :model/ContentTranslation :locale "de"))))+ (is (= 1 (t2/count :model/ContentTranslation :locale "de"))))))++(deftest values-that-bind-as-several-arguments-test+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "x" :msgstr "ex"}+ :model/ContentTranslation _ {:locale "fr" :msgid "y" :msgstr "why"}]+ (testing "a marked collection spreads across one argument per element"+ (is (= #{"ex" "why"}+ (set (map :msgstr (t2/select :model/ContentTranslation+ {:where [:in :locale [:auto/param ["de" "fr"]]]}))))))+ (testing "the same value marked twice binds twice"+ (is (= ["ex"]+ (map :msgstr (t2/select :model/ContentTranslation+ {:where [:and+ [:= :locale [:auto/param "de"]]+ [:not= :msgid [:auto/param "de"]]]})))))+ (testing "marked and unmarked values sit side by side"+ (is (= ["ex"]+ (map :msgstr (t2/select :model/ContentTranslation+ {:where [:and [:= :locale [:auto/param "de"]] [:= :msgid "x"]]})))))))++(deftest unmarked-string-is-already-a-literal-test+ (testing "an unmarked string was never the danger -- HoneySQL binds it"+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "a" :msgstr "b"}]+ (is (empty? (t2/select :model/ContentTranslation {:where [:= :locale "de' OR '1'='1"]})))+ (is (seq (t2/select :model/ContentTranslation {:where [:= :locale "de"]}))))))diff --git a/test/metabase/app_db/value_guard_test.clj b/test/metabase/app_db/value_guard_test.cljnew file mode 100644index 00000000000..76bbf015e42--- /dev/null+++ b/test/metabase/app_db/value_guard_test.clj@@ -0,0 +1,74 @@+(ns metabase.app-db.value-guard-test+ (:require+ [clojure.test :refer :all]+ [honey.sql :as sql]+ [metabase.app-db.value-guard :as value-guard]))++(defn- formatted+ "Lift the markers in `query` and compile it, as the app-DB compile step does."+ [query]+ (let [[form params] (#'value-guard/auto-param query)]+ (sql/format form {:params params})))++(deftest ^:parallel binds-a-marked-value-test+ (is (= ["WHERE a = ?" 5]+ (formatted {:where [:= :a [:auto/param 5]]}))))++(deftest ^:parallel binds-a-value-that-would-otherwise-compile-as-sql-test+ (testing "the lift binds a map rather than letting HoneySQL compile it"+ ;; Note this exercises the lift alone. Through the pipeline a `{:raw ...}` never reaches it:+ ;; `honeysql-guard` runs `:before` this `:around` and rejects the payload while it is still+ ;; inline. See `metabase.app-db.params-test` for what a query actually does with one.+ (let [payload {:raw "(SELECT password FROM core_user)"}]+ (is (= ["WHERE a = ?" payload]+ (formatted {:where [:= :a [:auto/param payload]]}))))))++(deftest ^:parallel binds-each-marker-separately-test+ (is (= ["WHERE (a = ?) AND (b = ?)" 1 2]+ (formatted {:where [:and [:= :a [:auto/param 1]] [:= :b [:auto/param 2]]]}))))++(deftest ^:parallel marks-a-value-shaped-like-a-marker-test+ (testing "a marker's payload is the value, even when the payload is itself marker-shaped"+ (is (= ["WHERE a = ?" [:auto/param 5]]+ (formatted {:where [:= :a [:auto/param [:auto/param 5]]]})))))++(deftest ^:parallel param-keys-cannot-be-named-by-request-data-test+ (testing "a [:param k] arriving in a value slot cannot name a slot this query minted"+ (let [from-request [:param :p1]+ [form params] (#'value-guard/auto-param+ {:where [:and+ [:= :locale [:auto/param "secret"]]+ [:= :msgid from-request]]})]+ (is (thrown-with-msg? Exception #"missing parameter value"+ (sql/format form {:params params}))))))++(deftest ^:parallel leaves-a-map-keyed-by-the-marker-alone-test+ (testing "a map entry is also a two-element vector, so a map keyed by the marker must survive"+ (are [query] (= [query {}] (#'value-guard/auto-param query))+ {:auto/param 5}+ {:where [:= :a 1] :set {:auto/param "x"}})))++(deftest ^:parallel rejects-a-malformed-marker-test+ (testing "a marker written any way but [:auto/param value] is a mistake, not a value"+ (are [query] (thrown-with-msg? clojure.lang.ExceptionInfo #"Malformed"+ (#'value-guard/auto-param query))+ {:where [:= :a [:auto/param]]}+ {:where [:= :a [:auto/param 1 2 3]]}+ {:where [:= :a (list :auto/param 5)]})))++(deftest ^:parallel binds-a-marked-kv-arg-test+ (testing "Toucan folds a marked kv-arg into [:auto/param column value]; it comes back as a comparison"+ (is (= ["WHERE locale = ?" "de"]+ (formatted {:where [:auto/param :locale "de"]}))))+ (testing "a marked kv-arg holding nil still compares as IS NULL"+ (is (= ["WHERE locale IS NULL"]+ (formatted {:where [:auto/param :locale nil]})))))++(deftest ^:parallel leaves-nil-to-honeysql-test+ (testing "a bound nil compares as `= ?`, which no row satisfies -- leave it a literal so it is IS NULL"+ (is (= (sql/format {:where [:= :a nil]})+ (formatted {:where [:= :a [:auto/param nil]]})))))++(deftest ^:parallel leaves-an-unmarked-query-alone-test+ (let [query {:select [:*] :from [:t] :where [:= :a 1]}]+ (is (= [query {}] (#'value-guard/auto-param query)))))diff --git a/test/metabase/content_translation/db_test.clj b/test/metabase/content_translation/db_test.cljnew file mode 100644index 00000000000..3d29a7c7e4b--- /dev/null+++ b/test/metabase/content_translation/db_test.clj@@ -0,0 +1,17 @@+(ns metabase.content-translation.db-test+ (:require+ [clojure.test :refer :all]+ [metabase.content-translation.db :as content-translation.db]+ [metabase.test :as mt]))++(deftest translations-for-locale-test+ (testing "filters to the requested locale"+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "hello" :msgstr "hallo"}+ :model/ContentTranslation _ {:locale "fr" :msgid "hello" :msgstr "bonjour"}]+ (is (= ["hallo"] (map :msgstr (content-translation.db/translations-for-locale "de"))))))+ (testing "a locale that looks like SQL is compared as a string, matching nothing"+ (mt/with-temp [:model/ContentTranslation _ {:locale "de" :msgid "hello" :msgstr "hallo"}]+ (is (empty? (content-translation.db/translations-for-locale "de' OR '1'='1")))))+ (testing "a non-string locale is rejected before it reaches the query"+ (is (thrown? clojure.lang.ExceptionInfo+ (content-translation.db/translations-for-locale {:raw "(SELECT 1)"})))))