Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Better snippet reference checking (#83685)

Nathan Voxlandd8a1f3adf8ed
diff --git a/src/metabase/driver/sql/parameters/substitute.clj b/src/metabase/driver/sql/parameters/substitute.clj
index 27e785f5716..9d73704d922 100644
--- a/src/metabase/driver/sql/parameters/substitute.clj
+++ b/src/metabase/driver/sql/parameters/substitute.clj
@@ -6,6 +6,7 @@
[metabase.driver-api.core :as driver-api]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.lib.core :as lib]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.literal :as lib.schema.literal]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.store :as qp.store]
@@ -19,6 +20,28 @@
(declare #^:private substitute*)
+(def ^:private max-substituted-sql-length
+ "Maximum length of the SQL produced by substituting parameters into a native query."
+ 10000000)
+
+(def ^:private max-snippet-expansions
+ "Maximum number of native query snippets that can be expanded while substituting parameters into a native query."
+ 10000)
+
+(defn- check-snippet-expansion! [{:keys [expanding-snippet-ids snippet-expansions]} {:keys [snippet-id]}]
+ (when (contains? expanding-snippet-ids snippet-id)
+ (throw (ex-info (tru "This query has circular referencing snippets.")
+ {:type driver-api/qp.error-type.invalid-query, :snippet-id snippet-id})))
+ (when (> (vswap! snippet-expansions inc) max-snippet-expansions)
+ (throw (ex-info (tru "This query uses too many snippets.")
+ {:type driver-api/qp.error-type.invalid-query}))))
+
+(defn- check-sql-length! [[sql :as acc]]
+ (when (> (count sql) max-substituted-sql-length)
+ (throw (ex-info (tru "This query is too long.")
+ {:type driver-api/qp.error-type.invalid-query})))
+ acc)
+
(defn- ->replacement-snippet-info [metadata-providerable x]
(qp.store/with-metadata-provider metadata-providerable
(sql.params.substitution/->replacement-snippet-info driver/*driver* x)))
@@ -30,6 +53,13 @@
#_args [:maybe [:sequential ::lib.schema.literal/param-value]]
#_missing [:maybe [:sequential :string]]]])
+(mr/def ::ctx
+ [:map {:closed true}
+ ;; IDs of the snippets currently being expanded, used to detect snippets that reference themselves
+ [:expanding-snippet-ids [:set ::lib.schema.id/snippet]]
+ ;; volatile count of the snippets expanded so far by the current call to [[substitute]]
+ [:snippet-expansions [:fn volatile?]]])
+
(mu/defn- substitute-field-param :- ::acc
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
[sql args missing] :- ::acc
@@ -58,13 +88,16 @@
(mu/defn- substitute-native-query-snippet :- ::acc
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ ctx :- ::ctx
param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]
[sql args missing] :- ::acc
in-optional? :- :boolean
v :- :metabase.lib.parameters.parse.types/referenced-query-snippet]
+ (check-snippet-expansion! ctx v)
(let [{:keys [replacement-snippet]} (->replacement-snippet-info metadata-providerable v)
[processed-snippet snippet-args snippet-missing] (substitute*
metadata-providerable
+ (update ctx :expanding-snippet-ids conj (:snippet-id v))
param->value
(lib/parse-parameters replacement-snippet)
in-optional?)]
@@ -74,6 +107,7 @@
(mu/defn- substitute-param :- ::acc
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ ctx :- ::ctx
param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]
[sql args missing] :- ::acc
in-optional? :- :boolean
@@ -91,7 +125,7 @@
(substitute-simple-query metadata-providerable [sql args missing] v)
(lib/parsed-referenced-query-snippet-param? v)
- (substitute-native-query-snippet metadata-providerable param->value [sql args missing] in-optional? v)
+ (substitute-native-query-snippet metadata-providerable ctx param->value [sql args missing] in-optional? v)
(= v lib/parsed-param-no-value-placeholder)
[sql args (conj missing k)]
@@ -103,10 +137,11 @@
(mu/defn- substitute-optional :- ::acc
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ ctx :- ::ctx
param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]
[sql args missing] :- ::acc
{subclauses :args} :- :metabase.lib.parameters.parse.types/optional]
- (let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable param->value subclauses true)]
+ (let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable ctx param->value subclauses true)]
(if (seq opt-missing)
[sql args missing]
[(str sql opt-sql) (concat args opt-args) missing])))
@@ -114,23 +149,25 @@
(mu/defn- substitute* :- ::acc
"Returns a sequence of `[replaced-sql-string jdbc-args missing-parameters]`."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ ctx :- ::ctx
param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]
parsed :- [:sequential :metabase.lib.parameters.parse/parsed-token]
in-optional? :- :boolean]
(reduce
(fn [[sql args missing] x]
- (cond
- (string? x)
- [(str sql x) args missing]
+ (check-sql-length!
+ (cond
+ (string? x)
+ [(str sql x) args missing]
- (lib/parsed-param? x)
- (substitute-param metadata-providerable param->value [sql args missing] in-optional? x)
+ (lib/parsed-param? x)
+ (substitute-param metadata-providerable ctx param->value [sql args missing] in-optional? x)
- (lib/parsed-optional-param? x)
- (substitute-optional metadata-providerable param->value [sql args missing] x)
+ (lib/parsed-optional-param? x)
+ (substitute-optional metadata-providerable ctx param->value [sql args missing] x)
- :else
- (throw (ex-info (format "Unexpected parsed param ^%s %s" (some-> x class .getCanonicalName) (pr-str x)) {:x x}))))
+ :else
+ (throw (ex-info (format "Unexpected parsed param ^%s %s" (some-> x class .getCanonicalName) (pr-str x)) {:x x})))))
nil
parsed))
@@ -147,7 +184,11 @@
param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]]
(log/tracef "Substituting %d params in query" (count param->value))
(let [[sql args missing] (try
- (substitute* metadata-providerable param->value parsed-query false)
+ (substitute* metadata-providerable
+ {:expanding-snippet-ids #{}, :snippet-expansions (volatile! 0)}
+ param->value
+ parsed-query
+ false)
(catch Throwable e
(throw (ex-info (tru "Unable to substitute parameters: {0}" (ex-message e))
{:type (or (:type (ex-data e)) driver-api/qp.error-type.qp)
diff --git a/src/metabase/query_processor/middleware/resolve_referenced.clj b/src/metabase/query_processor/middleware/resolve_referenced.clj
index ceb89cebd8c..dd8917fddcf 100644
--- a/src/metabase/query_processor/middleware/resolve_referenced.clj
+++ b/src/metabase/query_processor/middleware/resolve_referenced.clj
@@ -66,16 +66,22 @@
(as-> graph <>
(reduce (card-recurse mp node-type node-id) <> card-ids)
(reduce (snippet-recurse mp node-type node-id) <> snippet-ids)))
+ ;; a node that is already in the graph has already been walked (or is being walked), so we only need to add
+ ;; the new edge (which throws if it creates a cycle) instead of walking it again
+ (depend-and-walk [graph node dependency walk]
+ (let [visited? (seq (dep/immediate-dependents graph dependency))
+ graph (dep/depend graph node dependency)]
+ (if visited?
+ graph
+ (walk graph))))
(card-recurse [mp node-type id]
(fn [graph nested-card-id]
- (card-subquery-graph mp
- (dep/depend graph [node-type id] [::card nested-card-id])
- nested-card-id)))
+ (depend-and-walk graph [node-type id] [::card nested-card-id]
+ #(card-subquery-graph mp % nested-card-id))))
(snippet-recurse [mp node-type id]
(fn [graph nested-snippet-id]
- (snippet-subquery-graph mp
- (dep/depend graph [node-type id] [::snippet nested-snippet-id])
- nested-snippet-id)))]
+ (depend-and-walk graph [node-type id] [::snippet nested-snippet-id]
+ #(snippet-subquery-graph mp % nested-snippet-id))))]
(let [card-ids (lib/native-query-card-ids init-query)
snippet-ids (lib/native-query-snippet-ids init-query)]
(subquery-graph* metadata-providerable
diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.clj
index 8fb41473a4e..f1f5e80efa2 100644
--- a/test/metabase/driver/sql/parameters/substitute_test.clj
+++ b/test/metabase/driver/sql/parameters/substitute_test.clj
@@ -386,6 +386,56 @@
(substitute query {"snippet: outer" (lib/parsed-referenced-query-snippet-param 123 "{{snippet:symbol_is_A}}")
"snippet: symbol_is_A" (lib/parsed-referenced-query-snippet-param 124 "symbol = 'A'")}))))))
+(deftest ^:parallel substitute-repeated-native-query-snippets-test
+ (testing "The same snippet can be used more than once, including from within other snippets"
+ (let [query ["SELECT * FROM test_scores WHERE " (lib/parsed-param "snippet: outer")
+ " OR " (lib/parsed-param "snippet: symbol_is_A")]]
+ (is (=? ["SELECT * FROM test_scores WHERE (symbol = 'A' AND symbol = 'A') OR symbol = 'A'" nil]
+ (substitute query {"snippet: outer" (lib/parsed-referenced-query-snippet-param
+ 123 "({{snippet:symbol_is_A}} AND {{snippet:symbol_is_A}})")
+ "snippet: symbol_is_A" (lib/parsed-referenced-query-snippet-param 124 "symbol = 'A'")}))))))
+
+(deftest ^:parallel substitute-circular-native-query-snippets-test
+ (testing "Snippets that (directly or indirectly) reference themselves are rejected"
+ (let [query ["SELECT * FROM test_scores WHERE " (lib/parsed-param "snippet: a")]]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"circular"
+ (substitute query {"snippet: a" (lib/parsed-referenced-query-snippet-param 1 "{{snippet: a}}")})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"circular"
+ (substitute query {"snippet: a" (lib/parsed-referenced-query-snippet-param 1 "x {{snippet: b}}")
+ "snippet: b" (lib/parsed-referenced-query-snippet-param 2 "y {{snippet: a}}")}))))))
+
+(defn- doubling-snippets
+ "Snippets `s0` ... `s<depth>` where each references the next one twice, and the last one has `leaf-content`."
+ [depth leaf-content]
+ (into {(str "snippet: s" depth) (lib/parsed-referenced-query-snippet-param (inc depth) leaf-content)}
+ (for [i (range depth)
+ :let [next-tag (str "{{snippet: s" (inc i) "}}")]]
+ [(str "snippet: s" i) (lib/parsed-referenced-query-snippet-param (inc i) (str next-tag next-tag))])))
+
+(deftest ^:parallel substitute-native-query-snippets-expansion-limit-test
+ (testing "Snippets that expand into too many nested snippets are rejected"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"too many"
+ (substitute ["SELECT " (lib/parsed-param "snippet: s0")]
+ (doubling-snippets 20 ""))))))
+
+(deftest ^:parallel substitute-native-query-snippets-length-limit-test
+ (testing "Snippets that expand into a huge query are rejected"
+ (let [leaf (str/join (repeat 10000 "x"))]
+ (is (= (+ (count "SELECT ") (* 512 (count leaf)))
+ (count (first (substitute ["SELECT " (lib/parsed-param "snippet: s0")]
+ (doubling-snippets 9 leaf))))))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"too long"
+ (substitute ["SELECT " (lib/parsed-param "snippet: s0")]
+ (doubling-snippets 10 leaf)))))))
+
;;; ------------------------------------------ simple substitution — {{x}} ------------------------------------------
(defn- substitute-e2e [sql params]
diff --git a/test/metabase/query_processor/middleware/resolve_referenced_test.clj b/test/metabase/query_processor/middleware/resolve_referenced_test.clj
index ef9c75039f2..f69b374fad9 100644
--- a/test/metabase/query_processor/middleware/resolve_referenced_test.clj
+++ b/test/metabase/query_processor/middleware/resolve_referenced_test.clj
@@ -366,3 +366,33 @@
(testing "Should NOT throw an exception for valid non-cyclic chain"
;; This should complete, without throwing an exception, and return a dependency graph
(is (some? (#'qp.resolve-referenced/check-for-circular-references entrypoint-query))))))))
+
+(deftest shared-snippet-references-are-visited-once-test
+ (testing "Snippets reachable through many paths are only walked once when checking for circular references"
+ (let [depth 10
+ snippet-id (fn [level which] (+ 100 (* 2 level) which))
+ snippet-name (fn [level which] (str "snippet-" level "-" which))
+ snippets (for [level (range (inc depth))
+ which [0 1]
+ :let [refs (when (< level depth)
+ [[(snippet-id (inc level) 0) (snippet-name (inc level) 0)]
+ [(snippet-id (inc level) 1) (snippet-name (inc level) 1)]])]]
+ (make-snippet {:id (snippet-id level which)
+ :name (snippet-name level which)
+ :content (apply str "x" (for [[_ ref-name] refs]
+ (str " {{snippet: " ref-name "}}")))
+ :snippet-refs refs}))
+ metadata-provider (lib.tu/mock-metadata-provider meta/metadata-provider {:native-query-snippets snippets})
+ query (lib/query
+ metadata-provider
+ {:database (meta/id)
+ :type :native
+ :native {:query "SELECT {{snippet: snippet-0-0}}"
+ :template-tags (make-snippet-template-tag (snippet-id 0 0) (snippet-name 0 0))}})
+ fetches (atom 0)
+ orig lib.metadata/native-query-snippet]
+ (mt/with-dynamic-fn-redefs [lib.metadata/native-query-snippet (fn [mp id]
+ (swap! fetches inc)
+ (orig mp id))]
+ (#'qp.resolve-referenced/check-for-circular-references query))
+ (is (< @fetches (* 2 (count snippets)))))))