Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Lint values, not call style, and write the value-binding rubric down (#82686)

bryanc03c02a9a1be
diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
index e3f3a4b30f6..5facc634235 100644
--- a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
@@ -31,7 +31,8 @@
(def ^:private value-operators
"Operators whose second argument is a value rather than a column."
- '#{= not= < > <= >= like not-like ilike not-ilike in not-in between})
+ '#{= not= != <> is-distinct-from is-not-distinct-from
+ < > <= >= like not-like ilike not-ilike in not-in between not-between})
(defn- marked?
"Whether `node` is an `[:auto/param v]` marker."
@@ -39,28 +40,138 @@
(and (hooks/vector-node? node)
(= :auto/param (some-> (first (:children node)) hooks/sexpr))))
+(defn- distinct-by
+ "`coll` with only the first element for each distinct `(f element)`."
+ [f coll]
+ (->> coll (reduce (fn [[seen acc] x]
+ (let [k (f x)]
+ (if (contains? seen k) [seen acc] [(conj seen k) (conj acc x)])))
+ [#{} []])
+ second))
+
+(def ^:private identifier-clauses
+ "Clause keys whose entries name a column or a table rather than carrying a value.
+
+ A symbol here is a name, and `metabase.app-db.value-guard` refuses a marker in one, so reporting
+ it would ask for a fix that throws at compile. `:order-by` and `:group-by` are included even
+ though the guard allows a marker there: it compiles to `ORDER BY ?` and loses the ordering, so
+ asking for one is still wrong."
+ #{:select :select-distinct :select-top :from :join :left-join :right-join :inner-join :full-join
+ :cross-join :update :insert-into :delete-from :returning :with :with-columns :using
+ :order-by :group-by :partition-by :window})
+
+(def ^:private expression-entry-clauses
+ "Identifier clauses where index 0 of an `[expr alias]` entry is an expression, not a name.
+
+ HoneySQL binds a parameter there -- `SELECT ? AS a` -- so the guard accepts a marker, and mirrors
+ this set as `expression-entry-clauses` in `metabase.app-db.value-guard`. Everything after index 0
+ is an alias, and a bare entry is a name; the guard refuses a marker in both."
+ #{:select :select-distinct :select-top :returning :partition-by})
+
+(defn- comparison-node?
+ "Whether `node` is a keyword-headed comparison -- the one shape inside an identifier clause that
+ does hold values, as in the computed projection `[[:= :engine v] :is_match]`."
+ [node]
+ (and (hooks/vector-node? node)
+ (let [op (some-> (first (:children node)) hooks/sexpr)]
+ (and (keyword? op) (contains? value-operators (symbol (name op)))))))
+
+(declare value-nodes)
+
+(defn- identifier-clause-value-nodes
+ "The value slots reachable inside an identifier clause: a subquery's own clauses, and the
+ arguments of a comparison. The names themselves are not candidates."
+ [node]
+ (cond
+ (hooks/map-node? node) (value-nodes node)
+ (comparison-node? node) (value-nodes node)
+ (hooks/vector-node? node) (mapcat identifier-clause-value-nodes (:children node))
+ (hooks/list-node? node) (mapcat identifier-clause-value-nodes (:children node))
+ :else nil))
+
+(defn- expression-entry-value-nodes
+ "The value slots of an [[expression-entry-clauses]] clause: index 0 of each `[expr alias]` entry."
+ [node]
+ (when (hooks/vector-node? node)
+ (mapcat (fn [entry]
+ (when (hooks/vector-node? entry)
+ (when-let [expr (first (:children entry))]
+ ;; Keep the expression AND descend, so a bare symbol is reported and a computed
+ ;; projection or a subquery sitting there has its own values checked.
+ (cons expr (value-nodes expr)))))
+ (:children node))))
+
+(defn- clause-value-nodes
+ "The value slots of the value under clause key `clause`."
+ [clause node]
+ (cond
+ (contains? expression-entry-clauses clause) (expression-entry-value-nodes node)
+ (contains? identifier-clauses clause) (identifier-clause-value-nodes node)
+ :else (value-nodes node)))
+
(defn- value-nodes
"The nodes sitting in a value slot of `node`, following the clause shapes a query map uses."
[node]
(cond
- ;; A query map -- walk its clause values, not its keys.
+ ;; A query map -- walk its clause values, not its keys. A clause that names columns or tables
+ ;; contributes only what is nested inside it.
(hooks/map-node? node)
- (mapcat value-nodes (take-nth 2 (rest (:children node))))
+ (mapcat (fn [[k v]]
+ (if (hooks/keyword-node? k)
+ (clause-value-nodes (hooks/sexpr k) v)
+ (value-nodes v)))
+ (partition 2 (:children node)))
(hooks/vector-node? node)
(let [[head & args] (:children node)
op (some-> head hooks/sexpr)]
(cond
+ ;; A marker IS the value slot -- never descend into its payload, or the marked value
+ ;; gets reported as though it were bare.
+ (marked? node)
+ nil
+
(contains? #{:and :or :not} op)
(mapcat value-nodes args)
;; `[:= col v]`, `[:in col vs]`, `[:between col lo hi]` -- the first argument is the
- ;; column and the rest are values, whatever the arity.
+ ;; column and the rest are values, whatever the arity. Keep each value AND descend into it,
+ ;; so a subquery in a value slot -- `[:in :id {:select ... :where [:= :z z]}]` -- has its
+ ;; own values checked too. Descending alone would drop the plain-token case.
(and (keyword? op) (contains? value-operators (symbol (name op))))
- (rest args)
+ (mapcat #(cons % (value-nodes %)) (rest args))
+ ;; A function-call form -- `[:lower v]` -- or a literal collection -- `[a b]` in an `:in`.
+ ;; Both hold values, so yield the children themselves as candidates as well as
+ ;; descending, since a token child returns nothing on its own.
:else
- (mapcat value-nodes (:children node))))
+ (mapcat #(cons % (value-nodes %)) (:children node))))
+
+ ;; A clause built conditionally -- `(when flag [:= :col v])`, `(if ... )`, `(cond-> ...)`. The
+ ;; value slots are inside, so walk the children rather than stopping. Without this a value
+ ;; wrapped in `when` is invisible to the check, which is a false negative in a security lint.
+ ;;
+ ;; An `assoc` call gets its clause context back: `(assoc q :order-by [[col dir]])` is the same
+ ;; clause as `{:order-by [[col dir]]}`, but written outside a map literal, where the walk would
+ ;; otherwise fall back to shape alone and report the column and the direction as values.
+ ;; `cond->` is covered because the walk descends into the nested `assoc`.
+ (hooks/list-node? node)
+ (let [children (vec (:children node))
+ assoc? (= 'assoc (some-> (first children) hooks/sexpr))
+ ;; index of a clause value -> its clause key
+ clauses (when assoc?
+ (into {} (keep-indexed (fn [i child]
+ (when (and (hooks/keyword-node? child)
+ (contains? identifier-clauses (hooks/sexpr child))
+ (< (inc i) (count children)))
+ [(inc i) (hooks/sexpr child)]))
+ children)))]
+ (mapcat (fn [i child]
+ (if-let [clause (get clauses i)]
+ (clause-value-nodes clause child)
+ (value-nodes child)))
+ (range)
+ children))
:else nil))
@@ -77,35 +188,102 @@
(let [f (some-> (first (:children node)) hooks/sexpr)]
(not (contains? write-fns (some-> f name symbol)))))
-(defn- kv-arg-pair
- "The first `:column value` pair a query call passes after its model, if any.
+(def ^:private clause-keys
+ "Query-map clause keys. A map keyed by these is a query map rather than a map of conditions."
+ #{:select :select-distinct :from :where :join :left-join :right-join :inner-join :full-join
+ :cross-join :group-by :having :order-by :limit :offset :for :union :union-all :with
+ :with-columns :returning :values :set})
+
+(defn- query-map-node?
+ "Whether `node` is a query map rather than a map of column/value conditions.
+
+ `(t2/delete! :model/X {:where [:= :k k]})` passes a query map, whose values [[value-nodes]]
+ already walks -- treating it as conditions too would report every value twice."
+ [node]
+ (boolean (some #(and (hooks/keyword-node? %) (contains? clause-keys (hooks/sexpr %)))
+ (take-nth 2 (:children node)))))
+
+(def ^:private changes-map-fns
+ "Conditions-map fns whose arglist also ends in a changes map, so a lone trailing map is changes."
+ '#{update! update-or-insert!})
+
+(def ^:private conditions-map-fns
+ "Calls whose first argument after the model is a map of CONDITIONS rather than a query map.
+
+ `(t2/update! :model/X {:key k} {:v 1})` filters on `k`, so it is a where-clause value -- but it
+ reaches the query as a plain map entry rather than a `:where` clause, so neither the query-map
+ walker nor the kv-arg walker sees it. It is a shape the sweep leaves in place, so it has to be
+ checked here."
+ '#{update! update-or-insert! delete!})
+
+(defn- conditions-map-value-nodes
+ "The values of a conditions map passed to one of [[conditions-map-fns]]."
+ [f args]
+ (when (contains? conditions-map-fns (some-> f name symbol))
+ (let [after-model (->> args
+ (drop-while #(not (and (hooks/keyword-node? %)
+ (= "model" (namespace (hooks/sexpr %))))))
+ rest)]
+ ;; Only the map immediately after the model is conditions. A later map is the changes map,
+ ;; whose values are written rather than filtered on, and a written value is never marked: a
+ ;; column's `:in` transform runs on the marker itself and stores it as data.
+ ;; `update!`'s arglist ENDS in the changes map, so its first map is conditions only when
+ ;; another argument follows: `(t2/update! model {:v written})` is the two-arity call whose
+ ;; single map is CHANGES, so flagging it would ask for a marker that corrupts the write.
+ ;; `delete!` has no changes map, so its map is always conditions.
+ (when-let [m (when (or (not (contains? changes-map-fns (some-> f name symbol)))
+ (next after-model))
+ (first after-model))]
+ (when (and (hooks/map-node? m) (not (query-map-node? m)))
+ (mapcat (fn [v]
+ ;; The column comes from the map key, so an operator form here holds only
+ ;; values -- `{:key [:in ks]}` -- exactly as a kv-arg pair does.
+ (if (and (hooks/vector-node? v) (not (marked? v)))
+ (let [[head & op-args] (:children v)
+ op (some-> head hooks/sexpr)]
+ (if (and (keyword? op) (contains? value-operators (symbol (name op))))
+ (mapcat #(cons % (value-nodes %)) op-args)
+ (cons v (value-nodes v))))
+ (cons v (value-nodes v))))
+ (take-nth 2 (rest (:children m)))))))))
+
+(defn- kv-arg-value-nodes
+ "The value nodes of a call written as `:column value` pairs.
`(t2/select :model/X :locale locale)`. Several of these fns take an argument before the model --
`(t2/select-one-fn :value :model/X :key k)` -- so the pairs do not start at a fixed offset; they
- start after the `:model/...` keyword. A call that does not name a literal model is not checked."
+ start after the `:model/...` keyword. A call that does not name a literal model is not checked.
+
+ A trailing query map is not a pair, and its values are reached by [[value-nodes]] instead. A
+ value that is itself an operator form -- `:id [:in ids]` -- has its own values walked, so the
+ collection inside is checked rather than the form."
[args]
(let [after-model (->> args
(drop-while #(not (and (hooks/keyword-node? %)
(= "model" (namespace (hooks/sexpr %))))))
- rest)
- [k v] after-model]
- (when (and k v (hooks/keyword-node? k))
- k)))
-
-(defn- lint-kv-args!
- "Register a finding for a query written as `:column value` pairs.
-
- Toucan builds those into the where clause itself, so the query's shape is not on the page: there
- is nothing for a reader, this linter, or the `.sql` extraction that follows this project to read.
- The same call takes a map -- `{:where [:= :locale v]}` for a select, `{:locale v}` for an
- `update!` -- which is the statement, near enough to read off."
- [node]
- (when-let [k (kv-arg-pair (rest (:children node)))]
- (hooks/reg-finding!
- (assoc (meta k)
- :message (format "Pass this query a map rather than `%s ...` pairs, so its shape is on the page."
- (hooks/sexpr k))
- :type :metabase/unsafe-app-db-query))))
+ rest)]
+ (->> after-model
+ ;; `partition-all` rather than `partition` so a trailing odd argument (a query map, which
+ ;; is not a pair) is still seen and skipped by the keyword-node? test below.
+ (partition-all 2)
+ (mapcat (fn [[k v]]
+ (when (and v (hooks/keyword-node? k))
+ (if-not (hooks/vector-node? v)
+ [v]
+ (let [[head & op-args] (:children v)
+ op (some-> head hooks/sexpr)]
+ (cond
+ ;; Already marked -- the marker is the value slot.
+ (marked? v) [v]
+
+ ;; `:id [:in ids]`. Toucan folds the column in from the pair key, so
+ ;; EVERY argument here is a value -- unlike a query-map clause, where
+ ;; the first argument is the column.
+ (and (keyword? op)
+ (contains? value-operators (symbol (name op))))
+ (mapcat #(cons % (value-nodes %)) op-args)
+
+ :else (cons v (value-nodes v)))))))))))
(defn- lint-unmarked-values!
"Register a finding for each argument of the enclosing function that reaches a value slot unmarked.
@@ -113,7 +291,15 @@
Only a symbol is reported. A literal cannot carry a request value, and a value built inside the
function is out of reach of a check that does not follow it across a call."
[node]
- (doseq [value (mapcat value-nodes (rest (:children node)))
+ (doseq [value (let [args (rest (:children node))
+ f (some-> (first (:children node)) hooks/sexpr)]
+ ;; The three walkers overlap on some shapes -- a multi-arg operator in a kv-arg
+ ;; reaches both `value-nodes` and `kv-arg-value-nodes` -- so dedupe by source
+ ;; position rather than trying to keep them disjoint.
+ (->> (concat (mapcat value-nodes args)
+ (kv-arg-value-nodes args)
+ (conditions-map-value-nodes f args))
+ (distinct-by #(select-keys (meta %) [:row :col]))))
:when (and (hooks/token-node? value)
(symbol? (hooks/sexpr value))
(not (marked? value)))]
@@ -141,6 +327,5 @@
(db-namespace? (modules/config input) ns)
(not (test-file? filename))
(read-call? node))
- (lint-unmarked-values! node)
- (lint-kv-args! node))
+ (lint-unmarked-values! node))
input)
diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj
index 1010799e58c..e1cda4d196f 100644
--- a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj
+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj
@@ -103,24 +103,101 @@
(is (empty? (lint-query-call '(t2/select :model/X {:where [:= :locale locale]})
'metabase.foo.db "test/metabase/foo/db_test.clj")))))
-(deftest ^:parallel kv-arg-style-test
- (testing "a query written as :column value pairs is flagged whatever the values are"
- (are [form] (=? [{:type :metabase/unsafe-app-db-query, :message #"Pass this query a map.*"}]
+(deftest ^:parallel kv-arg-style-is-not-flagged-test
+ (testing "call style is not linted: a query written as :column value pairs is left alone"
+ ;; Extraction to a .sql file is semantic rather than a transcription of the HoneySQL shape, so a
+ ;; query map is no closer to that target than kv-args are. Converting also skips Toucan's type
+ ;; transforms, which only run in kv-arg position -- that bug shipped twice.
+ (are [form] (empty? (filter #(re-find #"Pass this query a map" (str (:message %)))
+ (lint-query-call form 'metabase.foo.db)))
+ '(t2/select :model/X :locale [:auto/param locale])
+ '(t2/select :model/X :archived false)
+ '(t2/select-one :model/X :id (long id))
+ '(t2/select-one-fn :value :model/Setting :key [:auto/param k])
+ '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})))
+ (testing "an unmarked value in a kv-arg is still flagged -- only the style check went away"
+ (is (=? [{:type :metabase/unsafe-app-db-query, :message #".*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call '(t2/select :model/X :locale locale) 'metabase.foo.db)))))
+
+(deftest ^:parallel conditions-map-values-are-linted-test
+ (testing "a conditions map -- a shape the sweep leaves in place -- is checked"
+ ;; `(t2/update! model {:col v} changes)` filters on v, but it is not a `:where` clause, so
+ ;; neither the query-map walker nor the kv-arg walker used to see it. A namespace written this
+ ;; way passed the lint vacuously.
+ (are [form] (=? [{:type :metabase/unsafe-app-db-query
+ :message #".*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call form 'metabase.foo.db))
+ '(t2/update! :model/X {:key k} {:v 1})
+ '(t2/delete! :model/X {:key k})
+ '(t2/update! :model/X {:key [:in ks]} {:v 1})))
+ (testing "a marked or coerced conditions value is not flagged"
+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))
+ '(t2/update! :model/X {:key [:auto/param k]} {:v 1})
+ '(t2/update! :model/X {:id (long id)} {:v 1})))
+ (testing "only the map right after the model is conditions -- the changes map is exempt"
+ ;; rule 1: a written value must arrive as itself, so it is never flagged.
+ (is (empty? (lint-query-call '(t2/update! :model/X {:id (long id)} {:v written})
+ 'metabase.foo.db)))))
+
+(deftest ^:parallel subquery-values-are-linted-test
+ (testing "a value inside a subquery in a value slot is reached"
+ (is (=? [{:type :metabase/unsafe-app-db-query
+ :message #".*`z`.*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call
+ '(t2/select :model/X {:where [:in :id {:select [:id] :from [:y] :where [:= :z z]}]})
+ 'metabase.foo.db))))
+ (testing "the plain token case still works -- descending must not replace it"
+ (is (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call '(t2/select :model/X {:where [:= :k v]}) 'metabase.foo.db)))))
+
+(deftest ^:parallel conditionally-built-clause-test
+ (testing "a value slot inside a conditional form is still reached"
+ ;; `value-nodes` used to stop at a list node, so anything a `when`/`if`/`cond->` built was
+ ;; invisible -- a false negative in a security lint. permissions/db.clj and collections/db.clj
+ ;; both build clauses this way.
+ (are [form] (=? [{:type :metabase/unsafe-app-db-query
+ :message #".*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call form 'metabase.foo.db))
+ '(t2/select :model/X {:where [:and (when k [:= :key k])]})))
+ (testing "both branches of an if are reached"
+ (is (= 2 (count (lint-query-call
+ '(t2/select :model/X {:where [:and (if flag [:= :key k] [:= :key other-k])]})
+ 'metabase.foo.db)))))
+ (testing "a marked value inside a conditional form is not flagged"
+ (is (empty? (lint-query-call
+ '(t2/select :model/X {:where [:and (when k [:= :key [:auto/param k]])]})
+ 'metabase.foo.db)))))
+
+(deftest ^:parallel kv-arg-values-are-linted-test
+ (testing "an unmarked value in a kv-arg pair is flagged, so dropping the style check left no gap"
+ (are [form] (=? [{:type :metabase/unsafe-app-db-query
+ :message #".*reaches a SQL value slot unmarked.*"}]
(lint-query-call form 'metabase.foo.db))
'(t2/select :model/X :locale locale)
- '(t2/select :model/X :archived false)
+ '(t2/select-one :model/X :id id)
+ ;; the pairs do not start at a fixed offset -- :value comes before the model here
+ '(t2/select-one-fn :value :model/Setting :key k)
+ ;; a value inside an operator form is reached, rather than the form being treated as a value
+ '(t2/select :model/X :id [:in ids])))
+ (testing "a marked or coerced kv-arg value is not flagged"
+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))
'(t2/select :model/X :locale [:auto/param locale])
- '(t2/select-one :model/X :id (long id))))
- (testing "one finding per call, not one per pair"
- (is (= 1 (count (lint-query-call '(t2/select :model/X :locale locale :msgid msgid)
- 'metabase.foo.db)))))
- (testing "a fn with an argument before the model is still recognised"
- (is (=? [{:message #"Pass this query a map.*"}]
- (lint-query-call '(t2/select-one-fn :value :model/Setting :key k) 'metabase.foo.db))))
- (testing "a map query is not flagged"
+ '(t2/select-one :model/X :id (long id))
+ '(t2/select :model/X :id [:in (mapv long ids)])
+ '(t2/select :model/X :id [:in [:auto/param ids]])))
+ (testing "a literal kv-arg value is not flagged"
(are [form] (empty? (lint-query-call form 'metabase.foo.db))
- '(t2/select :model/X {:where [:= :locale [:auto/param locale]]})
- '(t2/update! :model/X {:locale [:auto/param locale]} {:msgstr "x"}))))
+ '(t2/select :model/X :archived false)
+ '(t2/select :model/X :status "pending")
+ '(t2/select :model/X :engine :appdb)))
+ (testing "one unmarked value per pair is reported"
+ (is (= 2 (count (lint-query-call '(t2/select :model/X :locale locale :msgid msgid)
+ 'metabase.foo.db)))))
+ (testing "a call mixing kv-args with a trailing query map reports each value exactly once"
+ ;; The two walkers run over the same arguments, so this pins that they do not double-report.
+ (is (= 2 (count (lint-query-call
+ '(t2/select :model/X :locale locale {:where [:= :other other]})
+ 'metabase.foo.db))))))
(deftest ^:parallel write-calls-are-not-linted-for-values-test
(testing "an insert's values are written, not filtered on, so they are not flagged"
@@ -136,3 +213,112 @@
(testing "a value operator with an unexpected arity still has its args examined"
(is (seq (lint-query-call '(t2/select :model/X {:where [:= :a b c]}) 'metabase.foo.db)))
(is (seq (lint-query-call '(t2/select :model/X {:where [:between :a lo hi]}) 'metabase.foo.db)))))
+
+(deftest ^:parallel operator-coverage-test
+ (testing "comparison operators HoneySQL accepts as aliases for not= are checked"
+ ;; The set used to list only `not=`, so `[:!= :col v]` -- live in collections/db.clj -- was
+ ;; silently unchecked even though HoneySQL binds it as `<>`.
+ (are [form] (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call form 'metabase.foo.db))
+ '(t2/select :model/X {:where [:!= :k v]})
+ '(t2/select :model/X {:where [:<> :k v]})
+ '(t2/select :model/X {:where [:is-distinct-from :k v]})))
+ (testing "a two-value operator reports both values"
+ (is (= 2 (count (lint-query-call '(t2/select :model/X {:where [:not-between :k lo hi]})
+ 'metabase.foo.db)))))
+ (testing "a value wrapped in a function-call form is reached"
+ ;; `[:lower v]` compiles to `LOWER(?)`, so `v` is a real bind slot.
+ (are [form] (seq (lint-query-call form 'metabase.foo.db))
+ '(t2/select :model/X {:where [:= :k [:lower v]]})
+ '(t2/select :model/X {:where [:= :k [:cast v :text]]})
+ '(t2/select :model/X :k [:lower v])))
+ (testing "values in a literal collection are reached"
+ ;; `[:in :k [a b]]` binds each element -- `IN (?, ?)`.
+ (is (= 2 (count (lint-query-call '(t2/select :model/X {:where [:in :k [a b]]})
+ 'metabase.foo.db))))))
+
+(deftest ^:parallel no-duplicate-findings-test
+ (testing "a multi-arg operator in a kv-arg reports each value exactly once"
+ ;; value-nodes misreads `[:between lo hi]` as a clause (lo as column), while
+ ;; kv-arg-value-nodes reads it correctly, so the walkers overlap on `hi`. Deduping by source
+ ;; position makes that harmless.
+ (is (= 2 (count (lint-query-call '(t2/select :model/X :d [:between lo hi]) 'metabase.foo.db))))
+ (is (= 2 (count (lint-query-call '(t2/delete! :model/X {:d [:between lo hi]})
+ 'metabase.foo.db)))))
+ (testing "a query map passed where conditions go reports each value once"
+ ;; `{:where ...}` is a query map that value-nodes already walks; treating it as conditions too
+ ;; reported everything twice. Live in mcp/db.clj, metabot/db.clj, testing_api/db.clj.
+ (is (= 1 (count (lint-query-call '(t2/delete! :model/X {:where [:= :k k]}) 'metabase.foo.db))))
+ (is (= 2 (count (lint-query-call '(t2/update! :model/X {:where [:and [:= :a a] [:= :b b]]} {:v 1})
+ 'metabase.foo.db))))))
+
+(deftest ^:parallel two-arity-update-changes-map-test
+ (testing "update!'s lone trailing map is CHANGES, not conditions, so its values are not flagged"
+ ;; Toucan's arglist ends in the changes map. remote_sync/db.clj's mark-all-rsos-synced! is this
+ ;; shape; flagging `written` would ask for a marker on a written value, which the column's
+ ;; `:in` transform stores as data.
+ (is (empty? (lint-query-call '(t2/update! :model/X {:v written}) 'metabase.foo.db))))
+ (testing "delete! has no changes map, so its lone map is always conditions"
+ (is (=? [{:message #"`k`.*"}]
+ (lint-query-call '(t2/delete! :model/X {:k k}) 'metabase.foo.db))))
+ (testing "update!'s FIRST map is still conditions when a changes map follows"
+ (is (=? [{:message #"`k`.*"}]
+ (lint-query-call '(t2/update! :model/X {:k k} {:v 1}) 'metabase.foo.db)))))
+
+(deftest ^:parallel identifier-clauses-are-not-value-slots-test
+ (testing "a symbol naming a column or a table is not reported"
+ ;; `value-guard` refuses a marker in these clauses, so reporting one asks for a fix that throws
+ ;; at compile. data_studio/db.clj, task_history/db.clj and permissions/db.clj are written this
+ ;; way. `:select` is the exception: index 0 of its entries is an expression, so it has its own
+ ;; test in expression-entry-clauses-test.
+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))
+ '(t2/query {:select [:*] :from [[union-query :subquery]]})
+ '(t2/query {:select [:*] :from [:t] :order-by [[:sort_key sort-direction]]})
+ '(t2/query {:select [:*] :from [:t] :group-by [group-column]})))
+ (testing "a value nested inside an identifier clause is still reported"
+ (are [form] (=? [{:message #"`v`.*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call form 'metabase.foo.db))
+ ;; a computed projection is a real comparison
+ '(t2/query {:select [[[:= :engine v] :is_match]] :from [:t]})
+ ;; so is a CASE sort key
+ '(t2/query {:select [:*] :from [:t] :order-by [[[:case [:= :a v] 1 :else 2] :asc]]})
+ ;; a subquery carries its own where clause
+ '(t2/query {:select [:*] :from [[{:select [:*] :from [:u] :where [:= :k v]} :s]]}))))
+
+(deftest ^:parallel expression-entry-clauses-test
+ (testing "index 0 of an entry is a value slot, because HoneySQL binds a parameter there"
+ ;; `{:select [[[:auto/param "a"] :k]]}` compiles to `SELECT ? AS "K"`, so `value-guard` accepts
+ ;; a marker there. Treating the whole clause as names would hide a request value.
+ (are [form] (=? [{:type :metabase/unsafe-app-db-query
+ :message #"`v`.*reaches a SQL value slot unmarked.*"}]
+ (lint-query-call form 'metabase.foo.db))
+ '(t2/query {:select [[v :label]] :from [:t]})
+ '(t2/query {:select-distinct [[v :label]] :from [:t]})
+ '(t2/query {:insert-into [:t] :returning [[v :label]]})))
+ (testing "the alias, a bare entry and a table entry are names"
+ ;; `value-guard` refuses a marker in each of these, so asking for one would break the query.
+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))
+ '(t2/query {:select [[:key alias-sym]] :from [:t]})
+ '(t2/query {:select [cols] :from [:t]})
+ '(t2/query {:select [:*] :from [[tbl :x]]})))
+ (testing "a computed projection and a subquery entry still reach their values"
+ (are [form] (=? [{:message #"`v`.*"}] (lint-query-call form 'metabase.foo.db))
+ '(t2/query {:select [[[:= :engine v] :is_match]] :from [:t]})
+ '(t2/query {:select [:*] :from [[{:select [:*] :from [:u] :where [:= :k v]} :s]]}))))
+
+(deftest ^:parallel clause-context-survives-assoc-test
+ (testing "a clause key in an assoc call gives its value the clause context back"
+ ;; task_history/db.clj:174 and five other namespaces build `:order-by` this way. Without this
+ ;; the lint asks for a marker on a column name and on a sort direction, and the fix the message
+ ;; names loses the ordering with no error.
+ (are [form] (empty? (lint-query-call form 'metabase.foo.db))
+ '(t2/query (assoc q :order-by [[sort-column sort-direction] [:id :desc]]))
+ '(t2/query (cond-> q flag (assoc :order-by [[sort-column sort-direction]])))
+ '(t2/query (assoc q :group-by [group-column]))
+ '(t2/query (assoc q :select [[:key alias-sym]]))))
+ (testing "a value clause built the same way is still reported"
+ (is (=? [{:message #"`v`.*"}]
+ (lint-query-call '(t2/query (assoc q :where [:= :k v])) 'metabase.foo.db))))
+ (testing "an expression entry built the same way is still reported"
+ (is (=? [{:message #"`v`.*"}]
+ (lint-query-call '(t2/query (assoc q :select [[v :label]])) 'metabase.foo.db)))))
diff --git a/src/metabase/app_db/value_guard.clj b/src/metabase/app_db/value_guard.clj
index ac91cb5df2e..3efdfc190e3 100644
--- a/src/metabase/app_db/value_guard.clj
+++ b/src/metabase/app_db/value_guard.clj
@@ -99,6 +99,125 @@
[]
(keyword (str "p" (Long/toUnsignedString (.nextLong (ThreadLocalRandom/current)) 36))))
+(def ^:private identifier-clauses
+ "Query-map keys whose entries name columns or tables rather than carrying values."
+ ;; `:order-by` and `:group-by` are deliberately absent: HoneySQL binds a param in both
+ ;; (`ORDER BY ?`), so a marker there is a pointless no-op rather than a dropped value, and
+ ;; refusing it would turn a harmless mistake into a production exception.
+ #{:select :select-distinct :select-top :from :join :left-join :right-join :inner-join :full-join
+ :cross-join :update :insert-into :delete-from :returning :with :with-columns :using
+ :create-table :drop-table :truncate :partition-by :window})
+
+(def ^:private on-condition-clauses
+ "Join clauses that alternate a table with an ON condition, so odd positions hold values."
+ ;; `:cross-join` is NOT one of these -- it takes a flat list of tables with no condition, so
+ ;; every element is an identifier.
+ #{:join :left-join :right-join :inner-join :full-join})
+
+(def ^:private expression-entry-clauses
+ "Identifier clauses where index 0 of an `[expr alias]` entry is an expression, so HoneySQL binds a
+ param there."
+ ;; Each of these formats `[[:param :k] :a]` as `? AS a` or `? a`. The table clauses bind it too --
+ ;; `FROM ? AS a` -- but a table cannot be a parameter, so the database rejects the statement, and
+ ;; refusing it here names the mistake instead. `:with` throws inside HoneySQL, and the DDL clauses
+ ;; format it as an identifier or inline the value.
+ #{:select :select-distinct :select-top :returning :partition-by})
+
+(declare marker-in-identifier-position?)
+
+(defn- marker-in-entry?
+ "Whether a marker sits in an identifier position of `entry`, one element of identifier `clause`.
+
+ An entry is `expr`, or `[expr alias]`. HoneySQL tells those apart POSITIONALLY, not by shape, so
+ only index 0 may be an expression -- everything after it is an alias, which is always an
+ identifier. Checking by head shape instead let `[:t [:auto/param \"al\"]]` pass as though `:t`
+ headed an operator form, and the value was silently compiled to the identifier `param`.
+
+ A marker directly at index 0 is bound, not dropped, in a clause in [[expression-entry-clauses]], so
+ it does not count there."
+ [clause entry]
+ (cond
+ ;; The entry IS a marker -- `:select [[:auto/param "n"]]`. Destructuring it below would read
+ ;; `:auto/param` as the expression, so catch it before that.
+ (marker-form? entry) true
+ (not (sequential? entry)) (marker-in-identifier-position? entry)
+ :else
+ (let [[expr & aliases] entry]
+ (boolean (or (and (not (and (marker-form? expr)
+ (contains? expression-entry-clauses clause)))
+ (marker-in-identifier-position? expr))
+ ;; An alias slot can only be a name. A marker anywhere in one is a mistake.
+ ;; A nested query map is not an alias -- `:with` pairs a name with a query --
+ ;; so those are scanned separately by [[query-maps]].
+ (some #(and (not (map? %)) (contains-marker? %)) aliases))))))
+
+(defn- marker-in-identifier-position?
+ "Whether a marker sits in an identifier slot within `form`.
+
+ Descends past two things, which are not mistakes:
+
+ - a keyword-headed operator form, whose arguments are values. A computed projection --
+ `[[:= :engine [:auto/param \"h2\"]] :is_match]` -- puts a real comparison in a clause that
+ otherwise holds identifiers.
+ - a nested query map. A subquery has its own clauses, and [[check-marker-placement]] scans it
+ separately, so a marker in its `:where` is judged there rather than here."
+ [form]
+ (cond
+ (map? form) false
+ (marker-form? form) true
+ (operator-form? form) false
+ (sequential? form) (boolean (some marker-in-identifier-position? form))
+ :else false))
+
+(defn- query-maps
+ "`query` and every map nested anywhere inside it.
+
+ A subquery's clauses have to be scanned on their own terms: `:select` in an outer query holds
+ identifiers, but a subquery sitting there has its own `:where` that holds values -- and its own
+ `:select`, which holds identifiers again."
+ [query]
+ (let [found (volatile! [])]
+ (walk/postwalk (fn [x] (when (map? x) (vswap! found conj x)) x) query)
+ @found))
+
+(defn- misplaced-marker
+ "The offending entry of `clause`, if `v` puts a marker in an identifier position, else nil."
+ [clause v]
+ (let [entries (cond
+ ;; The whole value is a marker -- `:from [:auto/param "t"]`. Split as a list, it
+ ;; would be `:auto/param` and the payload, and neither is a marker.
+ (marker-form? v) [v]
+
+ ;; Alternating table / ON condition -- only the table halves are identifiers.
+ (and (contains? on-condition-clauses clause) (sequential? v))
+ (take-nth 2 v)
+
+ ;; Every other identifier clause holds a flat list of entries -- including
+ ;; `:cross-join`, which takes tables with no ON condition.
+ (sequential? v) v
+
+ ;; A bare value is its own single entry -- `:update :some_table`.
+ :else [v])]
+ (first (filter #(marker-in-entry? clause %) entries))))
+
+(defn- check-marker-placement
+ "Refuse a marker sitting in a clause that names columns or tables, in `query` or any subquery."
+ [query]
+ ;; `contains-marker?` first: the overwhelming majority of app-DB queries carry no marker at all,
+ ;; and this runs on every one of them at compile.
+ (when (contains-marker? query)
+ (doseq [m (query-maps query)
+ [clause v] m
+ :when (contains? identifier-clauses clause)
+ :let [part (misplaced-marker clause v)]
+ :when part]
+ (throw (ex-info (str "[:auto/param ...] in a " clause " clause: " (pr-str part)
+ ". That slot names a column or table, so the marker would compile to the"
+ " identifier `param` and the value would be dropped. A marker belongs in"
+ " a value slot.")
+ {:type ::marker-outside-value-slot, :clause clause, :form part
+ :query query})))))
+
(defn- auto-param
"Rewrite `[:auto/param v]` markers in `query` into HoneySQL's `[:param :kN]`, returning
`[rewritten-query params-map]`.
@@ -165,7 +284,8 @@
(if-not (map? built-query)
(do (assert-no-marker-survived! built-query)
(next-method query-type model built-query))
- (let [[query params] (auto-param built-query)]
+ (let [_ (check-marker-placement built-query)
+ [query params] (auto-param built-query)]
(assert-no-marker-survived! query)
(if-not (seq params)
(next-method query-type model query)
diff --git a/test/metabase/app_db/value_guard_test.clj b/test/metabase/app_db/value_guard_test.clj
index 76bbf015e42..ee194ee9a40 100644
--- a/test/metabase/app_db/value_guard_test.clj
+++ b/test/metabase/app_db/value_guard_test.clj
@@ -72,3 +72,77 @@
(deftest ^:parallel leaves-an-unmarked-query-alone-test
(let [query {:select [:*] :from [:t] :where [:= :a 1]}]
(is (= [query {}] (#'value-guard/auto-param query)))))
+
+(deftest ^:parallel refuses-a-marker-outside-a-value-slot-test
+ (testing "a marker in a clause that names columns or tables is refused"
+ ;; Without this the lift rewrites the marker wherever it sits, and HoneySQL formats the result
+ ;; in an identifier slot as the literal identifier `param` -- dropping the value with no signal.
+ (are [query] (= ::value-guard/marker-outside-value-slot
+ (try (#'value-guard/check-marker-placement query) nil
+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))
+ {:select [[:auto/param "name"]] :from [:t]}
+ {:select [:*] :from [[[:auto/param "t"]]]}
+ ;; an ALIAS slot -- HoneySQL tells [expr alias] apart positionally, so a marker after
+ ;; index 0 is always an identifier. This compiled to `FROM t AS param k` before.
+ {:select [:*] :from [[:t [:auto/param "al"]]]}
+ {:select [[:a [:auto/param "al"]]] :from [:t]}
+ ;; :cross-join takes only tables -- no ON condition -- so every element is an identifier
+ {:select [:*] :from [:a] :cross-join [:b [:auto/param "c"]]}
+ {:create-table :t :with-columns [[[:auto/param "c"] :int]]}
+ ;; the whole clause value is a marker -- read as a list of entries, it splits into
+ ;; `:auto/param` and the payload, neither a marker. This compiled to `FROM param, k` before.
+ {:select [:*] :from [:auto/param "t"]}
+ {:update [:auto/param "t"] :set {:a 1}}
+ {:select [:*] :from [:t] :join [:auto/param "u"]}))
+ (testing ":order-by and :group-by bind a param, so a marker there is a no-op rather than a drop"
+ ;; `{:order-by [[[:param :k] :asc]]}` compiles to `ORDER BY ? ASC` with the value bound --
+ ;; nothing is discarded, so refusing it would turn a harmless mistake into an exception.
+ (are [query] (nil? (#'value-guard/check-marker-placement query))
+ {:select [:*] :from [:t] :order-by [[[:auto/param "a"] :asc]]}
+ {:select [:*] :from [:t] :group-by [[:auto/param "a"]]}))
+ (testing "a marker in a genuine value slot is left alone"
+ (are [query] (nil? (#'value-guard/check-marker-placement query))
+ {:select [:*] :from [:t] :where [:= :a [:auto/param 1]]}
+ {:where [:auto/param :locale "de"]}
+ ;; a join alternates table and ON condition; the condition is a value slot
+ {:select [:*] :from [:t] :join [:u [:= :t.a [:auto/param 1]]]}))
+ (testing "a marker in an expression inside an identifier clause is a real value slot"
+ ;; A computed projection or a CASE sort key puts a genuine comparison in a clause that
+ ;; otherwise holds identifiers. Refusing these would break a namespace that grows one later.
+ (are [query] (nil? (#'value-guard/check-marker-placement query))
+ {:select [[[:= :engine [:auto/param "h2"]] :is_match]] :from [:t]}
+ {:select [:*] :from [:t] :order-by [[[:case [:= :a [:auto/param 1]] 1 :else 2] :asc]]}
+ {:select [:*] :from [:t] :group-by [[:coalesce :a [:auto/param 1]]]}))
+ (testing "a subquery's OWN identifier clauses are scanned too"
+ ;; The scan recurses, so a marker one level down in a nested :select is still refused -- it
+ ;; would otherwise compile to the identifier `param` inside the subquery and drop the value.
+ (is (= ::value-guard/marker-outside-value-slot
+ (try (#'value-guard/check-marker-placement
+ {:select [[[:exists {:select [[:auto/param "name"]] :from [:t]}] :e]]})
+ nil
+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))))
+ (testing "a subquery has its own clauses, so an outer identifier slot holding one is not scanned"
+ ;; `t2/exists?` wraps the whole query in `:select [[[:exists {...}]]]`, and that inner map's
+ ;; `:where` is a real value slot. Scanning the outer `:select` wholesale would reject it.
+ (is (nil? (#'value-guard/check-marker-placement
+ {:select [[[:exists {:select [[[:inline 1]]]
+ :from [[:content_translation]]
+ :where [:auto/param :locale "de"]}] :exists]]})))))
+
+(deftest ^:parallel marker-at-index-0-of-an-entry-test
+ (testing "a marker directly at index 0 of an [expr alias] entry is bound where that index is an expression"
+ ;; `{:select [[[:param :k] :a]]}` compiles to `SELECT ? AS a` -- the value is bound, not dropped.
+ (are [query] (nil? (#'value-guard/check-marker-placement query))
+ {:select [[[:auto/param 1] :a]] :from [:t]}
+ {:select-distinct [[[:auto/param 1] :a]] :from [:t]}
+ {:delete-from :t :returning [[[:auto/param 1] :a]]})
+ (is (= ["SELECT ? AS a FROM t" 1]
+ (formatted {:select [[[:auto/param 1] :a]] :from [:t]}))))
+ (testing "a marker at index 0 of a table entry is still refused"
+ ;; HoneySQL binds it -- `FROM ? AS a` -- but a table cannot be a parameter, so the statement
+ ;; fails at the database. Refusing here names the mistake instead.
+ (are [query] (= ::value-guard/marker-outside-value-slot
+ (try (#'value-guard/check-marker-placement query) nil
+ (catch clojure.lang.ExceptionInfo e (:type (ex-data e)))))
+ {:select [:*] :from [[[:auto/param "t"] :a]]}
+ {:select [:*] :from [:t] :join [[[:auto/param "u"] :a] [:= 1 1]]})))