The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Lint values, not call style, and write the value-binding rubric down (#82686)
bryanc03c02a9a1be
diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.cljindex e3f3a4b30f6..5facc634235 100644--- a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj@@ -31,7 +31,8 @@(def ^:private value-operators"Operators whose second argument is a value rather than a column."- '#{= not= < > <= >= like not-like ilike not-ilike in not-in between})+ '#{= not= != <> is-distinct-from is-not-distinct-from+ < > <= >= like not-like ilike not-ilike in not-in between not-between})(defn- marked?"Whether `node` is an `[:auto/param v]` marker."@@ -39,28 +40,138 @@(and (hooks/vector-node? node)(= :auto/param (some-> (first (:children node)) hooks/sexpr))))+(defn- distinct-by+ "`coll` with only the first element for each distinct `(f element)`."+ [f coll]+ (->> coll (reduce (fn [[seen acc] x]+ (let [k (f x)]+ (if (contains? seen k) [seen acc] [(conj seen k) (conj acc x)])))+ [#{} []])+ second))++(def ^:private identifier-clauses+ "Clause keys whose entries name a column or a table rather than carrying a value.++ A symbol here is a name, and `metabase.app-db.value-guard` refuses a marker in one, so reporting+ it would ask for a fix that throws at compile. `:order-by` and `:group-by` are included even+ though the guard allows a marker there: it compiles to `ORDER BY ?` and loses the ordering, so+ asking for one is still wrong."+ #{:select :select-distinct :select-top :from :join :left-join :right-join :inner-join :full-join+ :cross-join :update :insert-into :delete-from :returning :with :with-columns :using+ :order-by :group-by :partition-by :window})++(def ^:private expression-entry-clauses+ "Identifier clauses where index 0 of an `[expr alias]` entry is an expression, not a name.++ HoneySQL binds a parameter there -- `SELECT ? AS a` -- so the guard accepts a marker, and mirrors+ this set as `expression-entry-clauses` in `metabase.app-db.value-guard`. Everything after index 0+ is an alias, and a bare entry is a name; the guard refuses a marker in both."+ #{:select :select-distinct :select-top :returning :partition-by})++(defn- comparison-node?+ "Whether `node` is a keyword-headed comparison -- the one shape inside an identifier clause that+ does hold values, as in the computed projection `[[:= :engine v] :is_match]`."+ [node]+ (and (hooks/vector-node? node)+ (let [op (some-> (first (:children node)) hooks/sexpr)]+ (and (keyword? op) (contains? value-operators (symbol (name op)))))))++(declare value-nodes)++(defn- identifier-clause-value-nodes+ "The value slots reachable inside an identifier clause: a subquery's own clauses, and the+ arguments of a comparison. The names themselves are not candidates."+ [node]+ (cond+ (hooks/map-node? node) (value-nodes node)+ (comparison-node? node) (value-nodes node)+ (hooks/vector-node? node) (mapcat identifier-clause-value-nodes (:children node))+ (hooks/list-node? node) (mapcat identifier-clause-value-nodes (:children node))+ :else nil))++(defn- expression-entry-value-nodes+ "The value slots of an [[expression-entry-clauses]] clause: index 0 of each `[expr alias]` entry."+ [node]+ (when (hooks/vector-node? node)+ (mapcat (fn [entry]+ (when (hooks/vector-node? entry)+ (when-let [expr (first (:children entry))]+ ;; Keep the expression AND descend, so a bare symbol is reported and a computed+ ;; projection or a subquery sitting there has its own values checked.+ (cons expr (value-nodes expr)))))+ (:children node))))++(defn- clause-value-nodes+ "The value slots of the value under clause key `clause`."+ [clause node]+ (cond+ (contains? expression-entry-clauses clause) (expression-entry-value-nodes node)+ (contains? identifier-clauses clause) (identifier-clause-value-nodes node)+ :else (value-nodes node)))+(defn- value-nodes"The nodes sitting in a value slot of `node`, following the clause shapes a query map uses."[node](cond- ;; A query map -- walk its clause values, not its keys.+ ;; A query map -- walk its clause values, not its keys. A clause that names columns or tables+ ;; contributes only what is nested inside it.(hooks/map-node? node)- (mapcat value-nodes (take-nth 2 (rest (:children node))))+ (mapcat (fn [[k v]]+ (if (hooks/keyword-node? k)+ (clause-value-nodes (hooks/sexpr k) v)+ (value-nodes v)))+ (partition 2 (:children node)))(hooks/vector-node? node)(let [[head & args] (:children node)op (some-> head hooks/sexpr)](cond+ ;; A marker IS the value slot -- never descend into its payload, or the marked value+ ;; gets reported as though it were bare.+ (marked? node)+ nil+(contains? #{:and :or :not} op)(mapcat value-nodes args);; `[:= col v]`, `[:in col vs]`, `[:between col lo hi]` -- the first argument is the- ;; column and the rest are values, whatever the arity.+ ;; column and the rest are values, whatever the arity. Keep each value AND descend into it,+ ;; so a subquery in a value slot -- `[:in :id {:select ... :where [:= :z z]}]` -- has its+ ;; own values checked too. Descending alone would drop the plain-token case.(and (keyword? op) (contains? value-operators (symbol (name op))))- (rest args)+ (mapcat #(cons % (value-nodes %)) (rest args))+ ;; A function-call form -- `[:lower v]` -- or a literal collection -- `[a b]` in an `:in`.+ ;; Both hold values, so yield the children themselves as candidates as well as+ ;; descending, since a token child returns nothing on its own.:else- (mapcat value-nodes (:children node))))+ (mapcat #(cons % (value-nodes %)) (:children node))))++ ;; A clause built conditionally -- `(when flag [:= :col v])`, `(if ... )`, `(cond-> ...)`. The+ ;; value slots are inside, so walk the children rather than stopping. Without this a value+ ;; wrapped in `when` is invisible to the check, which is a false negative in a security lint.+ ;;+ ;; An `assoc` call gets its clause context back: `(assoc q :order-by [[col dir]])` is the same+ ;; clause as `{:order-by [[col dir]]}`, but written outside a map literal, where the walk would+ ;; otherwise fall back to shape alone and report the column and the direction as values.+ ;; `cond->` is covered because the walk descends into the nested `assoc`.+ (hooks/list-node? node)+ (let [children (vec (:children node))+ assoc? (= 'assoc (some-> (first children) hooks/sexpr))+ ;; index of a clause value -> its clause key+ clauses (when assoc?+ (into {} (keep-indexed (fn [i child]+ (when (and (hooks/keyword-node? child)+ (contains? identifier-clauses (hooks/sexpr child))+ (< (inc i) (count children)))+ [(inc i) (hooks/sexpr child)]))+ children)))]+ (mapcat (fn [i child]+ (if-let [clause (get clauses i)]+ (clause-value-nodes clause child)+ (value-nodes child)))+ (range)+ children)):else nil))@@ -77,35 +188,102 @@(let [f (some-> (first (:children node)) hooks/sexpr)](not (contains? write-fns (some-> f name symbol)))))-(defn- kv-arg-pair- "The first `:column value` pair a query call passes after its model, if any.+(def ^:private clause-keys+ "Query-map clause keys. A map keyed by these is a query map rather than a map of conditions."+ #{:select :select-distinct :from :where :join :left-join :right-join :inner-join :full-join+ :cross-join :group-by :having :order-by :limit :offset :for :union :union-all :with+ :with-columns :returning :values :set})++(defn- query-map-node?+ "Whether `node` is a query map rather than a map of column/value conditions.++ `(t2/delete! :model/X {:where [:= :k k]})` passes a query map, whose values [[value-nodes]]+ already walks -- treating it as conditions too would report every value twice."+ [node]+ (boolean (some #(and (hooks/keyword-node? %) (contains? clause-keys (hooks/sexpr %)))+ (take-nth 2 (:children node)))))++(def ^:private changes-map-fns+ "Conditions-map fns whose arglist also ends in a changes map, so a lone trailing map is changes."+ '#{update! update-or-insert!})++(def ^:private conditions-map-fns+ "Calls whose first argument after the model is a map of CONDITIONS rather than a query map.++ `(t2/update! :model/X {:key k} {:v 1})` filters on `k`, so it is a where-clause value -- but it+ reaches the query as a plain map entry rather than a `:where` clause, so neither the query-map+ walker nor the kv-arg walker sees it. It is a shape the sweep leaves in place, so it has to be+ checked here."+ '#{update! update-or-insert! delete!})++(defn- conditions-map-value-nodes+ "The values of a conditions map passed to one of [[conditions-map-fns]]."+ [f args]+ (when (contains? conditions-map-fns (some-> f name symbol))+ (let [after-model (->> args+ (drop-while #(not (and (hooks/keyword-node? %)+ (= "model" (namespace (hooks/sexpr %))))))+ rest)]+ ;; Only the map immediately after the model is conditions. A later map is the changes map,+ ;; whose values are written rather than filtered on, and a written value is never marked: a+ ;; column's `:in` transform runs on the marker itself and stores it as data.+ ;; `update!`'s arglist ENDS in the changes map, so its first map is conditions only when+ ;; another argument follows: `(t2/update! model {:v written})` is the two-arity call whose+ ;; single map is CHANGES, so flagging it would ask for a marker that corrupts the write.+ ;; `delete!` has no changes map, so its map is always conditions.+ (when-let [m (when (or (not (contains? changes-map-fns (some-> f name symbol)))+ (next after-model))+ (first after-model))]+ (when (and (hooks/map-node? m) (not (query-map-node? m)))+ (mapcat (fn [v]+ ;; The column comes from the map key, so an operator form here holds only+ ;; values -- `{:key [:in ks]}` -- exactly as a kv-arg pair does.+ (if (and (hooks/vector-node? v) (not (marked? v)))+ (let [[head & op-args] (:children v)+ op (some-> head hooks/sexpr)]+ (if (and (keyword? op) (contains? value-operators (symbol (name op))))+ (mapcat #(cons % (value-nodes %)) op-args)+ (cons v (value-nodes v))))+ (cons v (value-nodes v))))+ (take-nth 2 (rest (:children m)))))))))++(defn- kv-arg-value-nodes+ "The value nodes of a call written as `:column value` pairs.`(t2/select :model/X :locale locale)`. Several of these fns take an argument before the model --`(t2/select-one-fn :value :model/X :key k)` -- so the pairs do not start at a fixed offset; they- start after the `:model/...` keyword. A call that does not name a literal model is not checked."+ start after the `:model/...` keyword. A call that does not name a literal model is not checked.++ A trailing query map is not a pair, and its values are reached by [[value-nodes]] instead. A+ value that is itself an operator form -- `:id [:in ids]` -- has its own values walked, so the+ collection inside is checked rather than the form."[args](let [after-model (->> args(drop-while #(not (and (hooks/keyword-node? %)(= "model" (namespace (hooks/sexpr %))))))- rest)- [k v] after-model]- (when (and k v (hooks/keyword-node? k))- k)))--(defn- lint-kv-args!- "Register a finding for a query written as `:column value` pairs.-- Toucan builds those into the where clause itself, so the query's shape is not on the page: there- is nothing for a reader, this linter, or the `.sql` extraction that follows this project to read.- The same call takes a map -- `{:where [:= :locale v]}` for a select, `{:locale v}` for an- `update!` -- which is the statement, near enough to read off."- [node]- (when-let [k (kv-arg-pair (rest (:children node)))]- (hooks/reg-finding!- (assoc (meta k)- :message (format "Pass this query a map rather than `%s ...` pairs, so its shape is on the page."- (hooks/sexpr k))- :type :metabase/unsafe-app-db-query))))+ rest)]+ (->> after-model+ ;; `partition-all` rather than `partition` so a trailing odd argument (a query map, which+ ;; is not a pair) is still seen and skipped by the keyword-node? test below.+ (partition-all 2)+ (mapcat (fn [[k v]]+ (when (and v (hooks/keyword-node? k))+ (if-not (hooks/vector-node? v)+ [v]+ (let [[head & op-args] (:children v)+ op (some-> head hooks/sexpr)]+ (cond+ ;; Already marked -- the marker is the value slot.+ (marked? v) [v]++ ;; `:id [:in ids]`. Toucan folds the column in from the pair key, so+ ;; EVERY argument here is a value -- unlike a query-map clause, where+ ;; the first argument is the column.+ (and (keyword? op)+ (contains? value-operators (symbol (name op))))+ (mapcat #(cons % (value-nodes %)) op-args)++ :else (cons v (value-nodes v)))))))))))(defn- lint-unmarked-values!"Register a finding for each argument of the enclosing function that reaches a value slot unmarked.@@ -113,7 +291,15 @@Only a symbol is reported. A literal cannot carry a request value, and a value built inside thefunction is out of reach of a check that does not follow it across a call."[node]- (doseq [value (mapcat value-nodes (rest (:children node)))+ (doseq [value (let [args (rest (:children node))+ f (some-> (first (:children node)) hooks/sexpr)]+ ;; The three walkers overlap on some shapes -- a multi-arg operator in a kv-arg+ ;; reaches both `value-nodes` and `kv-arg-value-nodes` -- so dedupe by source+ ;; position rather than trying to keep them disjoint.+ (->> (concat (mapcat value-nodes args)+ (kv-arg-value-nodes args)+ (conditions-map-value-nodes f args))+ (distinct-by #(select-keys (meta %) [:row :col])))):when (and (hooks/token-node? value)(symbol? (hooks/sexpr value))(not (marked? value)))]@@ -141,6 +327,5 @@(db-namespace? (modules/config input) ns)(not (test-file? filename))(read-call? node))- (lint-unmarked-values! node)- (lint-kv-args! node))+ (lint-unmarked-values! node))input)diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.cljindex 1010799e58c..e1cda4d196f 100644--- a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj@@ -103,24 +103,101 @@(is (empty? (lint-query-call '(t2/select :model/X {:where [:= :locale locale]})'metabase.foo.db "test/metabase/foo/db_test.clj")))))-(deftest ^:parallel kv-arg-style-test- (testing "a query written as :column value pairs is flagged whatever the values are"- (are [form] (=? [{:type :metabase/unsafe-app-db-query, :message #"Pass this query a map.*"}]+(deftest ^:parallel kv-arg-style-is-not-flagged-test+ (testing "call style is not linted: a query written as :column value pairs is left alone"+ ;; Extraction to a .sql file is semantic rather than a transcription of the HoneySQL shape, so a+ ;; query map is no closer to that target than kv-args are. Converting also skips Toucan's type+ ;; transforms, which only run in kv-arg position -- that bug shipped twice.+ (are [form] (empty? (filter #(re-find #"Pass this query a map" (str (:message %)))+ (lint-query-call form 'metabase.foo.db)))+ '(t2/select :model/X :locale [:auto/param locale])+ '(t2/select :model/X :archived false)+ '(t2/select-one :model/X :id (long id))+ '(t2/select-one-fn :value :model/Setting :key [:auto/param k])+ '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})))+ (testing "an unmarked value in a kv-arg is still flagged -- only the style check went away"+ (is (=? [{:type :metabase/unsafe-app-db-query, :message #".*reaches a SQL value slot unmarked.*"}]+ (lint-query-call '(t2/select :model/X :locale locale) 'metabase.foo.db)))))++(deftest ^:parallel conditions-map-values-are-linted-test+ (testing "a conditions map -- a shape the sweep leaves in place -- is checked"+ ;; `(t2/update! model {:col v} changes)` filters on v, but it is not a `:where` clause, so+ ;; neither the query-map walker nor the kv-arg walker used to see it. A namespace written this+ ;; way passed the lint vacuously.+ (are [form] (=? [{:type :metabase/unsafe-app-db-query+ :message #".*reaches a SQL value slot unmarked.*"}]+ (lint-query-call form 'metabase.foo.db))+ '(t2/update! :model/X {:key k} {:v 1})+ '(t2/delete! :model/X {:key k})+ '(t2/update! :model/X {:key [:in ks]} {:v 1})))+ (testing "a marked or coerced conditions value is not flagged"+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))+ '(t2/update! :model/X {:key [:auto/param k]} {:v 1})+ '(t2/update! :model/X {:id (long id)} {:v 1})))+ (testing "only the map right after the model is conditions -- the changes map is exempt"+ ;; rule 1: a written value must arrive as itself, so it is never flagged.+ (is (empty? (lint-query-call '(t2/update! :model/X {:id (long id)} {:v written})+ 'metabase.foo.db)))))++(deftest ^:parallel subquery-values-are-linted-test+ (testing "a value inside a subquery in a value slot is reached"+ (is (=? [{:type :metabase/unsafe-app-db-query+ :message #".*`z`.*reaches a SQL value slot unmarked.*"}]+ (lint-query-call+ '(t2/select :model/X {:where [:in :id {:select [:id] :from [:y] :where [:= :z z]}]})+ 'metabase.foo.db))))+ (testing "the plain token case still works -- descending must not replace it"+ (is (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]+ (lint-query-call '(t2/select :model/X {:where [:= :k v]}) 'metabase.foo.db)))))++(deftest ^:parallel conditionally-built-clause-test+ (testing "a value slot inside a conditional form is still reached"+ ;; `value-nodes` used to stop at a list node, so anything a `when`/`if`/`cond->` built was+ ;; invisible -- a false negative in a security lint. permissions/db.clj and collections/db.clj+ ;; both build clauses this way.+ (are [form] (=? [{:type :metabase/unsafe-app-db-query+ :message #".*reaches a SQL value slot unmarked.*"}]+ (lint-query-call form 'metabase.foo.db))+ '(t2/select :model/X {:where [:and (when k [:= :key k])]})))+ (testing "both branches of an if are reached"+ (is (= 2 (count (lint-query-call+ '(t2/select :model/X {:where [:and (if flag [:= :key k] [:= :key other-k])]})+ 'metabase.foo.db)))))+ (testing "a marked value inside a conditional form is not flagged"+ (is (empty? (lint-query-call+ '(t2/select :model/X {:where [:and (when k [:= :key [:auto/param k]])]})+ 'metabase.foo.db)))))++(deftest ^:parallel kv-arg-values-are-linted-test+ (testing "an unmarked value in a kv-arg pair is flagged, so dropping the style check left no gap"+ (are [form] (=? [{:type :metabase/unsafe-app-db-query+ :message #".*reaches a SQL value slot unmarked.*"}](lint-query-call form 'metabase.foo.db))'(t2/select :model/X :locale locale)- '(t2/select :model/X :archived false)+ '(t2/select-one :model/X :id id)+ ;; the pairs do not start at a fixed offset -- :value comes before the model here+ '(t2/select-one-fn :value :model/Setting :key k)+ ;; a value inside an operator form is reached, rather than the form being treated as a value+ '(t2/select :model/X :id [:in ids])))+ (testing "a marked or coerced kv-arg value is not flagged"+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))'(t2/select :model/X :locale [:auto/param locale])- '(t2/select-one :model/X :id (long id))))- (testing "one finding per call, not one per pair"- (is (= 1 (count (lint-query-call '(t2/select :model/X :locale locale :msgid msgid)- 'metabase.foo.db)))))- (testing "a fn with an argument before the model is still recognised"- (is (=? [{:message #"Pass this query a map.*"}]- (lint-query-call '(t2/select-one-fn :value :model/Setting :key k) 'metabase.foo.db))))- (testing "a map query is not flagged"+ '(t2/select-one :model/X :id (long id))+ '(t2/select :model/X :id [:in (mapv long ids)])+ '(t2/select :model/X :id [:in [:auto/param ids]])))+ (testing "a literal kv-arg value is not flagged"(are [form] (empty? (lint-query-call form 'metabase.foo.db))- '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})- '(t2/update! :model/X {:locale [:auto/param locale]} {:msgstr "x"}))))+ '(t2/select :model/X :archived false)+ '(t2/select :model/X :status "pending")+ '(t2/select :model/X :engine :appdb)))+ (testing "one unmarked value per pair is reported"+ (is (= 2 (count (lint-query-call '(t2/select :model/X :locale locale :msgid msgid)+ 'metabase.foo.db)))))+ (testing "a call mixing kv-args with a trailing query map reports each value exactly once"+ ;; The two walkers run over the same arguments, so this pins that they do not double-report.+ (is (= 2 (count (lint-query-call+ '(t2/select :model/X :locale locale {:where [:= :other other]})+ 'metabase.foo.db))))))(deftest ^:parallel write-calls-are-not-linted-for-values-test(testing "an insert's values are written, not filtered on, so they are not flagged"@@ -136,3 +213,112 @@(testing "a value operator with an unexpected arity still has its args examined"(is (seq (lint-query-call '(t2/select :model/X {:where [:= :a b c]}) 'metabase.foo.db)))(is (seq (lint-query-call '(t2/select :model/X {:where [:between :a lo hi]}) 'metabase.foo.db)))))++(deftest ^:parallel operator-coverage-test+ (testing "comparison operators HoneySQL accepts as aliases for not= are checked"+ ;; The set used to list only `not=`, so `[:!= :col v]` -- live in collections/db.clj -- was+ ;; silently unchecked even though HoneySQL binds it as `<>`.+ (are [form] (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]+ (lint-query-call form 'metabase.foo.db))+ '(t2/select :model/X {:where [:!= :k v]})+ '(t2/select :model/X {:where [:<> :k v]})+ '(t2/select :model/X {:where [:is-distinct-from :k v]})))+ (testing "a two-value operator reports both values"+ (is (= 2 (count (lint-query-call '(t2/select :model/X {:where [:not-between :k lo hi]})+ 'metabase.foo.db)))))+ (testing "a value wrapped in a function-call form is reached"+ ;; `[:lower v]` compiles to `LOWER(?)`, so `v` is a real bind slot.+ (are [form] (seq (lint-query-call form 'metabase.foo.db))+ '(t2/select :model/X {:where [:= :k [:lower v]]})+ '(t2/select :model/X {:where [:= :k [:cast v :text]]})+ '(t2/select :model/X :k [:lower v])))+ (testing "values in a literal collection are reached"+ ;; `[:in :k [a b]]` binds each element -- `IN (?, ?)`.+ (is (= 2 (count (lint-query-call '(t2/select :model/X {:where [:in :k [a b]]})+ 'metabase.foo.db))))))++(deftest ^:parallel no-duplicate-findings-test+ (testing "a multi-arg operator in a kv-arg reports each value exactly once"+ ;; value-nodes misreads `[:between lo hi]` as a clause (lo as column), while+ ;; kv-arg-value-nodes reads it correctly, so the walkers overlap on `hi`. Deduping by source+ ;; position makes that harmless.+ (is (= 2 (count (lint-query-call '(t2/select :model/X :d [:between lo hi]) 'metabase.foo.db))))+ (is (= 2 (count (lint-query-call '(t2/delete! :model/X {:d [:between lo hi]})+ 'metabase.foo.db)))))+ (testing "a query map passed where conditions go reports each value once"+ ;; `{:where ...}` is a query map that value-nodes already walks; treating it as conditions too+ ;; reported everything twice. Live in mcp/db.clj, metabot/db.clj, testing_api/db.clj.+ (is (= 1 (count (lint-query-call '(t2/delete! :model/X {:where [:= :k k]}) 'metabase.foo.db))))+ (is (= 2 (count (lint-query-call '(t2/update! :model/X {:where [:and [:= :a a] [:= :b b]]} {:v 1})+ 'metabase.foo.db))))))++(deftest ^:parallel two-arity-update-changes-map-test+ (testing "update!'s lone trailing map is CHANGES, not conditions, so its values are not flagged"+ ;; Toucan's arglist ends in the changes map. remote_sync/db.clj's mark-all-rsos-synced! is this+ ;; shape; flagging `written` would ask for a marker on a written value, which the column's+ ;; `:in` transform stores as data.+ (is (empty? (lint-query-call '(t2/update! :model/X {:v written}) 'metabase.foo.db))))+ (testing "delete! has no changes map, so its lone map is always conditions"+ (is (=? [{:message #"`k`.*"}]+ (lint-query-call '(t2/delete! :model/X {:k k}) 'metabase.foo.db))))+ (testing "update!'s FIRST map is still conditions when a changes map follows"+ (is (=? [{:message #"`k`.*"}]+ (lint-query-call '(t2/update! :model/X {:k k} {:v 1}) 'metabase.foo.db)))))++(deftest ^:parallel identifier-clauses-are-not-value-slots-test+ (testing "a symbol naming a column or a table is not reported"+ ;; `value-guard` refuses a marker in these clauses, so reporting one asks for a fix that throws+ ;; at compile. data_studio/db.clj, task_history/db.clj and permissions/db.clj are written this+ ;; way. `:select` is the exception: index 0 of its entries is an expression, so it has its own+ ;; test in expression-entry-clauses-test.+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))+ '(t2/query {:select [:*] :from [[union-query :subquery]]})+ '(t2/query {:select [:*] :from [:t] :order-by [[:sort_key sort-direction]]})+ '(t2/query {:select [:*] :from [:t] :group-by [group-column]})))+ (testing "a value nested inside an identifier clause is still reported"+ (are [form] (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]+ (lint-query-call form 'metabase.foo.db))+ ;; a computed projection is a real comparison+ '(t2/query {:select [[[:= :engine v] :is_match]] :from [:t]})+ ;; so is a CASE sort key+ '(t2/query {:select [:*] :from [:t] :order-by [[[:case [:= :a v] 1 :else 2] :asc]]})+ ;; a subquery carries its own where clause+ '(t2/query {:select [:*] :from [[{:select [:*] :from [:u] :where [:= :k v]} :s]]}))))++(deftest ^:parallel expression-entry-clauses-test+ (testing "index 0 of an entry is a value slot, because HoneySQL binds a parameter there"+ ;; `{:select [[[:auto/param "a"] :k]]}` compiles to `SELECT ? AS "K"`, so `value-guard` accepts+ ;; a marker there. Treating the whole clause as names would hide a request value.+ (are [form] (=? [{:type :metabase/unsafe-app-db-query+ :message #"`v`.*reaches a SQL value slot unmarked.*"}]+ (lint-query-call form 'metabase.foo.db))+ '(t2/query {:select [[v :label]] :from [:t]})+ '(t2/query {:select-distinct [[v :label]] :from [:t]})+ '(t2/query {:insert-into [:t] :returning [[v :label]]})))+ (testing "the alias, a bare entry and a table entry are names"+ ;; `value-guard` refuses a marker in each of these, so asking for one would break the query.+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))+ '(t2/query {:select [[:key alias-sym]] :from [:t]})+ '(t2/query {:select [cols] :from [:t]})+ '(t2/query {:select [:*] :from [[tbl :x]]})))+ (testing "a computed projection and a subquery entry still reach their values"+ (are [form] (=? [{:message #"`v`.*"}] (lint-query-call form 'metabase.foo.db))+ '(t2/query {:select [[[:= :engine v] :is_match]] :from [:t]})+ '(t2/query {:select [:*] :from [[{:select [:*] :from [:u] :where [:= :k v]} :s]]}))))++(deftest ^:parallel clause-context-survives-assoc-test+ (testing "a clause key in an assoc call gives its value the clause context back"+ ;; task_history/db.clj:174 and five other namespaces build `:order-by` this way. Without this+ ;; the lint asks for a marker on a column name and on a sort direction, and the fix the message+ ;; names loses the ordering with no error.+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))+ '(t2/query (assoc q :order-by [[sort-column sort-direction] [:id :desc]]))+ '(t2/query (cond-> q flag (assoc :order-by [[sort-column sort-direction]])))+ '(t2/query (assoc q :group-by [group-column]))+ '(t2/query (assoc q :select [[:key alias-sym]]))))+ (testing "a value clause built the same way is still reported"+ (is (=? [{:message #"`v`.*"}]+ (lint-query-call '(t2/query (assoc q :where [:= :k v])) 'metabase.foo.db))))+ (testing "an expression entry built the same way is still reported"+ (is (=? [{:message #"`v`.*"}]+ (lint-query-call '(t2/query (assoc q :select [[v :label]])) 'metabase.foo.db)))))diff --git a/src/metabase/app_db/value_guard.clj b/src/metabase/app_db/value_guard.cljindex ac91cb5df2e..3efdfc190e3 100644--- a/src/metabase/app_db/value_guard.clj+++ b/src/metabase/app_db/value_guard.clj@@ -99,6 +99,125 @@[](keyword (str "p" (Long/toUnsignedString (.nextLong (ThreadLocalRandom/current)) 36))))+(def ^:private identifier-clauses+ "Query-map keys whose entries name columns or tables rather than carrying values."+ ;; `:order-by` and `:group-by` are deliberately absent: HoneySQL binds a param in both+ ;; (`ORDER BY ?`), so a marker there is a pointless no-op rather than a dropped value, and+ ;; refusing it would turn a harmless mistake into a production exception.+ #{:select :select-distinct :select-top :from :join :left-join :right-join :inner-join :full-join+ :cross-join :update :insert-into :delete-from :returning :with :with-columns :using+ :create-table :drop-table :truncate :partition-by :window})++(def ^:private on-condition-clauses+ "Join clauses that alternate a table with an ON condition, so odd positions hold values."+ ;; `:cross-join` is NOT one of these -- it takes a flat list of tables with no condition, so+ ;; every element is an identifier.+ #{:join :left-join :right-join :inner-join :full-join})++(def ^:private expression-entry-clauses+ "Identifier clauses where index 0 of an `[expr alias]` entry is an expression, so HoneySQL binds a+ param there."+ ;; Each of these formats `[[:param :k] :a]` as `? AS a` or `? a`. The table clauses bind it too --+ ;; `FROM ? AS a` -- but a table cannot be a parameter, so the database rejects the statement, and+ ;; refusing it here names the mistake instead. `:with` throws inside HoneySQL, and the DDL clauses+ ;; format it as an identifier or inline the value.+ #{:select :select-distinct :select-top :returning :partition-by})++(declare marker-in-identifier-position?)++(defn- marker-in-entry?+ "Whether a marker sits in an identifier position of `entry`, one element of identifier `clause`.++ An entry is `expr`, or `[expr alias]`. HoneySQL tells those apart POSITIONALLY, not by shape, so+ only index 0 may be an expression -- everything after it is an alias, which is always an+ identifier. Checking by head shape instead let `[:t [:auto/param \"al\"]]` pass as though `:t`+ headed an operator form, and the value was silently compiled to the identifier `param`.++ A marker directly at index 0 is bound, not dropped, in a clause in [[expression-entry-clauses]], so+ it does not count there."+ [clause entry]+ (cond+ ;; The entry IS a marker -- `:select [[:auto/param "n"]]`. Destructuring it below would read+ ;; `:auto/param` as the expression, so catch it before that.+ (marker-form? entry) true+ (not (sequential? entry)) (marker-in-identifier-position? entry)+ :else+ (let [[expr & aliases] entry]+ (boolean (or (and (not (and (marker-form? expr)+ (contains? expression-entry-clauses clause)))+ (marker-in-identifier-position? expr))+ ;; An alias slot can only be a name. A marker anywhere in one is a mistake.+ ;; A nested query map is not an alias -- `:with` pairs a name with a query --+ ;; so those are scanned separately by [[query-maps]].+ (some #(and (not (map? %)) (contains-marker? %)) aliases))))))++(defn- marker-in-identifier-position?+ "Whether a marker sits in an identifier slot within `form`.++ Descends past two things, which are not mistakes:++ - a keyword-headed operator form, whose arguments are values. A computed projection --+ `[[:= :engine [:auto/param \"h2\"]] :is_match]` -- puts a real comparison in a clause that+ otherwise holds identifiers.+ - a nested query map. A subquery has its own clauses, and [[check-marker-placement]] scans it+ separately, so a marker in its `:where` is judged there rather than here."+ [form]+ (cond+ (map? form) false+ (marker-form? form) true+ (operator-form? form) false+ (sequential? form) (boolean (some marker-in-identifier-position? form))+ :else false))++(defn- query-maps+ "`query` and every map nested anywhere inside it.++ A subquery's clauses have to be scanned on their own terms: `:select` in an outer query holds+ identifiers, but a subquery sitting there has its own `:where` that holds values -- and its own+ `:select`, which holds identifiers again."+ [query]+ (let [found (volatile! [])]+ (walk/postwalk (fn [x] (when (map? x) (vswap! found conj x)) x) query)+ @found))++(defn- misplaced-marker+ "The offending entry of `clause`, if `v` puts a marker in an identifier position, else nil."+ [clause v]+ (let [entries (cond+ ;; The whole value is a marker -- `:from [:auto/param "t"]`. Split as a list, it+ ;; would be `:auto/param` and the payload, and neither is a marker.+ (marker-form? v) [v]++ ;; Alternating table / ON condition -- only the table halves are identifiers.+ (and (contains? on-condition-clauses clause) (sequential? v))+ (take-nth 2 v)++ ;; Every other identifier clause holds a flat list of entries -- including+ ;; `:cross-join`, which takes tables with no ON condition.+ (sequential? v) v++ ;; A bare value is its own single entry -- `:update :some_table`.+ :else [v])]+ (first (filter #(marker-in-entry? clause %) entries))))++(defn- check-marker-placement+ "Refuse a marker sitting in a clause that names columns or tables, in `query` or any subquery."+ [query]+ ;; `contains-marker?` first: the overwhelming majority of app-DB queries carry no marker at all,+ ;; and this runs on every one of them at compile.+ (when (contains-marker? query)+ (doseq [m (query-maps query)+ [clause v] m+ :when (contains? identifier-clauses clause)+ :let [part (misplaced-marker clause v)]+ :when part]+ (throw (ex-info (str "[:auto/param ...] in a " clause " clause: " (pr-str part)+ ". That slot names a column or table, so the marker would compile to the"+ " identifier `param` and the value would be dropped. A marker belongs in"+ " a value slot.")+ {:type ::marker-outside-value-slot, :clause clause, :form part+ :query query})))))+(defn- auto-param"Rewrite `[:auto/param v]` markers in `query` into HoneySQL's `[:param :kN]`, returning`[rewritten-query params-map]`.@@ -165,7 +284,8 @@(if-not (map? built-query)(do (assert-no-marker-survived! built-query)(next-method query-type model built-query))- (let [[query params] (auto-param built-query)]+ (let [_ (check-marker-placement built-query)+ [query params] (auto-param built-query)](assert-no-marker-survived! query)(if-not (seq params)(next-method query-type model query)diff --git a/test/metabase/app_db/value_guard_test.clj b/test/metabase/app_db/value_guard_test.cljindex 76bbf015e42..ee194ee9a40 100644--- a/test/metabase/app_db/value_guard_test.clj+++ b/test/metabase/app_db/value_guard_test.clj@@ -72,3 +72,77 @@(deftest ^:parallel leaves-an-unmarked-query-alone-test(let [query {:select [:*] :from [:t] :where [:= :a 1]}](is (= [query {}] (#'value-guard/auto-param query)))))++(deftest ^:parallel refuses-a-marker-outside-a-value-slot-test+ (testing "a marker in a clause that names columns or tables is refused"+ ;; Without this the lift rewrites the marker wherever it sits, and HoneySQL formats the result+ ;; in an identifier slot as the literal identifier `param` -- dropping the value with no signal.+ (are [query] (= ::value-guard/marker-outside-value-slot+ (try (#'value-guard/check-marker-placement query) nil+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))+ {:select [[:auto/param "name"]] :from [:t]}+ {:select [:*] :from [[[:auto/param "t"]]]}+ ;; an ALIAS slot -- HoneySQL tells [expr alias] apart positionally, so a marker after+ ;; index 0 is always an identifier. This compiled to `FROM t AS param k` before.+ {:select [:*] :from [[:t [:auto/param "al"]]]}+ {:select [[:a [:auto/param "al"]]] :from [:t]}+ ;; :cross-join takes only tables -- no ON condition -- so every element is an identifier+ {:select [:*] :from [:a] :cross-join [:b [:auto/param "c"]]}+ {:create-table :t :with-columns [[[:auto/param "c"] :int]]}+ ;; the whole clause value is a marker -- read as a list of entries, it splits into+ ;; `:auto/param` and the payload, neither a marker. This compiled to `FROM param, k` before.+ {:select [:*] :from [:auto/param "t"]}+ {:update [:auto/param "t"] :set {:a 1}}+ {:select [:*] :from [:t] :join [:auto/param "u"]}))+ (testing ":order-by and :group-by bind a param, so a marker there is a no-op rather than a drop"+ ;; `{:order-by [[[:param :k] :asc]]}` compiles to `ORDER BY ? ASC` with the value bound --+ ;; nothing is discarded, so refusing it would turn a harmless mistake into an exception.+ (are [query] (nil? (#'value-guard/check-marker-placement query))+ {:select [:*] :from [:t] :order-by [[[:auto/param "a"] :asc]]}+ {:select [:*] :from [:t] :group-by [[:auto/param "a"]]}))+ (testing "a marker in a genuine value slot is left alone"+ (are [query] (nil? (#'value-guard/check-marker-placement query))+ {:select [:*] :from [:t] :where [:= :a [:auto/param 1]]}+ {:where [:auto/param :locale "de"]}+ ;; a join alternates table and ON condition; the condition is a value slot+ {:select [:*] :from [:t] :join [:u [:= :t.a [:auto/param 1]]]}))+ (testing "a marker in an expression inside an identifier clause is a real value slot"+ ;; A computed projection or a CASE sort key puts a genuine comparison in a clause that+ ;; otherwise holds identifiers. Refusing these would break a namespace that grows one later.+ (are [query] (nil? (#'value-guard/check-marker-placement query))+ {:select [[[:= :engine [:auto/param "h2"]] :is_match]] :from [:t]}+ {:select [:*] :from [:t] :order-by [[[:case [:= :a [:auto/param 1]] 1 :else 2] :asc]]}+ {:select [:*] :from [:t] :group-by [[:coalesce :a [:auto/param 1]]]}))+ (testing "a subquery's OWN identifier clauses are scanned too"+ ;; The scan recurses, so a marker one level down in a nested :select is still refused -- it+ ;; would otherwise compile to the identifier `param` inside the subquery and drop the value.+ (is (= ::value-guard/marker-outside-value-slot+ (try (#'value-guard/check-marker-placement+ {:select [[[:exists {:select [[:auto/param "name"]] :from [:t]}] :e]]})+ nil+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))))+ (testing "a subquery has its own clauses, so an outer identifier slot holding one is not scanned"+ ;; `t2/exists?` wraps the whole query in `:select [[[:exists {...}]]]`, and that inner map's+ ;; `:where` is a real value slot. Scanning the outer `:select` wholesale would reject it.+ (is (nil? (#'value-guard/check-marker-placement+ {:select [[[:exists {:select [[[:inline 1]]]+ :from [[:content_translation]]+ :where [:auto/param :locale "de"]}] :exists]]})))))++(deftest ^:parallel marker-at-index-0-of-an-entry-test+ (testing "a marker directly at index 0 of an [expr alias] entry is bound where that index is an expression"+ ;; `{:select [[[:param :k] :a]]}` compiles to `SELECT ? AS a` -- the value is bound, not dropped.+ (are [query] (nil? (#'value-guard/check-marker-placement query))+ {:select [[[:auto/param 1] :a]] :from [:t]}+ {:select-distinct [[[:auto/param 1] :a]] :from [:t]}+ {:delete-from :t :returning [[[:auto/param 1] :a]]})+ (is (= ["SELECT ? AS a FROM t" 1]+ (formatted {:select [[[:auto/param 1] :a]] :from [:t]}))))+ (testing "a marker at index 0 of a table entry is still refused"+ ;; HoneySQL binds it -- `FROM ? AS a` -- but a table cannot be a parameter, so the statement+ ;; fails at the database. Refusing here names the mistake instead.+ (are [query] (= ::value-guard/marker-outside-value-slot+ (try (#'value-guard/check-marker-placement query) nil+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))+ {:select [:*] :from [[[:auto/param "t"] :a]]}+ {:select [:*] :from [:t] :join [[[:auto/param "u"] :a] [:= 1 1]]})))