The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Better snippet reference checking (#83685)
Nathan Voxlandd8a1f3adf8ed
diff --git a/src/metabase/driver/sql/parameters/substitute.clj b/src/metabase/driver/sql/parameters/substitute.cljindex 27e785f5716..9d73704d922 100644--- a/src/metabase/driver/sql/parameters/substitute.clj+++ b/src/metabase/driver/sql/parameters/substitute.clj@@ -6,6 +6,7 @@[metabase.driver-api.core :as driver-api][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.literal :as lib.schema.literal][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.store :as qp.store]@@ -19,6 +20,28 @@(declare #^:private substitute*)+(def ^:private max-substituted-sql-length+ "Maximum length of the SQL produced by substituting parameters into a native query."+ 10000000)++(def ^:private max-snippet-expansions+ "Maximum number of native query snippets that can be expanded while substituting parameters into a native query."+ 10000)++(defn- check-snippet-expansion! [{:keys [expanding-snippet-ids snippet-expansions]} {:keys [snippet-id]}]+ (when (contains? expanding-snippet-ids snippet-id)+ (throw (ex-info (tru "This query has circular referencing snippets.")+ {:type driver-api/qp.error-type.invalid-query, :snippet-id snippet-id})))+ (when (> (vswap! snippet-expansions inc) max-snippet-expansions)+ (throw (ex-info (tru "This query uses too many snippets.")+ {:type driver-api/qp.error-type.invalid-query}))))++(defn- check-sql-length! [[sql :as acc]]+ (when (> (count sql) max-substituted-sql-length)+ (throw (ex-info (tru "This query is too long.")+ {:type driver-api/qp.error-type.invalid-query})))+ acc)+(defn- ->replacement-snippet-info [metadata-providerable x](qp.store/with-metadata-provider metadata-providerable(sql.params.substitution/->replacement-snippet-info driver/*driver* x)))@@ -30,6 +53,13 @@#_args [:maybe [:sequential ::lib.schema.literal/param-value]]#_missing [:maybe [:sequential :string]]]])+(mr/def ::ctx+ [:map {:closed true}+ ;; IDs of the snippets currently being expanded, used to detect snippets that reference themselves+ [:expanding-snippet-ids [:set ::lib.schema.id/snippet]]+ ;; volatile count of the snippets expanded so far by the current call to [[substitute]]+ [:snippet-expansions [:fn volatile?]]])+(mu/defn- substitute-field-param :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable[sql args missing] :- ::acc@@ -58,13 +88,16 @@(mu/defn- substitute-native-query-snippet :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ ctx :- ::ctxparam->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]][sql args missing] :- ::accin-optional? :- :booleanv :- :metabase.lib.parameters.parse.types/referenced-query-snippet]+ (check-snippet-expansion! ctx v)(let [{:keys [replacement-snippet]} (->replacement-snippet-info metadata-providerable v)[processed-snippet snippet-args snippet-missing] (substitute*metadata-providerable+ (update ctx :expanding-snippet-ids conj (:snippet-id v))param->value(lib/parse-parameters replacement-snippet)in-optional?)]@@ -74,6 +107,7 @@(mu/defn- substitute-param :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ ctx :- ::ctxparam->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]][sql args missing] :- ::accin-optional? :- :boolean@@ -91,7 +125,7 @@(substitute-simple-query metadata-providerable [sql args missing] v)(lib/parsed-referenced-query-snippet-param? v)- (substitute-native-query-snippet metadata-providerable param->value [sql args missing] in-optional? v)+ (substitute-native-query-snippet metadata-providerable ctx param->value [sql args missing] in-optional? v)(= v lib/parsed-param-no-value-placeholder)[sql args (conj missing k)]@@ -103,10 +137,11 @@(mu/defn- substitute-optional :- ::acc[metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ ctx :- ::ctxparam->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]][sql args missing] :- ::acc{subclauses :args} :- :metabase.lib.parameters.parse.types/optional]- (let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable param->value subclauses true)]+ (let [[opt-sql opt-args opt-missing] (substitute* metadata-providerable ctx param->value subclauses true)](if (seq opt-missing)[sql args missing][(str sql opt-sql) (concat args opt-args) missing])))@@ -114,23 +149,25 @@(mu/defn- substitute* :- ::acc"Returns a sequence of `[replaced-sql-string jdbc-args missing-parameters]`."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ ctx :- ::ctxparam->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]parsed :- [:sequential :metabase.lib.parameters.parse/parsed-token]in-optional? :- :boolean](reduce(fn [[sql args missing] x]- (cond- (string? x)- [(str sql x) args missing]+ (check-sql-length!+ (cond+ (string? x)+ [(str sql x) args missing]- (lib/parsed-param? x)- (substitute-param metadata-providerable param->value [sql args missing] in-optional? x)+ (lib/parsed-param? x)+ (substitute-param metadata-providerable ctx param->value [sql args missing] in-optional? x)- (lib/parsed-optional-param? x)- (substitute-optional metadata-providerable param->value [sql args missing] x)+ (lib/parsed-optional-param? x)+ (substitute-optional metadata-providerable ctx param->value [sql args missing] x)- :else- (throw (ex-info (format "Unexpected parsed param ^%s %s" (some-> x class .getCanonicalName) (pr-str x)) {:x x}))))+ :else+ (throw (ex-info (format "Unexpected parsed param ^%s %s" (some-> x class .getCanonicalName) (pr-str x)) {:x x})))))nilparsed))@@ -147,7 +184,11 @@param->value :- [:maybe [:map-of :string :metabase.lib.parameters.parse.types/parsed-value]]](log/tracef "Substituting %d params in query" (count param->value))(let [[sql args missing] (try- (substitute* metadata-providerable param->value parsed-query false)+ (substitute* metadata-providerable+ {:expanding-snippet-ids #{}, :snippet-expansions (volatile! 0)}+ param->value+ parsed-query+ false)(catch Throwable e(throw (ex-info (tru "Unable to substitute parameters: {0}" (ex-message e)){:type (or (:type (ex-data e)) driver-api/qp.error-type.qp)diff --git a/src/metabase/query_processor/middleware/resolve_referenced.clj b/src/metabase/query_processor/middleware/resolve_referenced.cljindex ceb89cebd8c..dd8917fddcf 100644--- a/src/metabase/query_processor/middleware/resolve_referenced.clj+++ b/src/metabase/query_processor/middleware/resolve_referenced.clj@@ -66,16 +66,22 @@(as-> graph <>(reduce (card-recurse mp node-type node-id) <> card-ids)(reduce (snippet-recurse mp node-type node-id) <> snippet-ids)))+ ;; a node that is already in the graph has already been walked (or is being walked), so we only need to add+ ;; the new edge (which throws if it creates a cycle) instead of walking it again+ (depend-and-walk [graph node dependency walk]+ (let [visited? (seq (dep/immediate-dependents graph dependency))+ graph (dep/depend graph node dependency)]+ (if visited?+ graph+ (walk graph))))(card-recurse [mp node-type id](fn [graph nested-card-id]- (card-subquery-graph mp- (dep/depend graph [node-type id] [::card nested-card-id])- nested-card-id)))+ (depend-and-walk graph [node-type id] [::card nested-card-id]+ #(card-subquery-graph mp % nested-card-id))))(snippet-recurse [mp node-type id](fn [graph nested-snippet-id]- (snippet-subquery-graph mp- (dep/depend graph [node-type id] [::snippet nested-snippet-id])- nested-snippet-id)))]+ (depend-and-walk graph [node-type id] [::snippet nested-snippet-id]+ #(snippet-subquery-graph mp % nested-snippet-id))))](let [card-ids (lib/native-query-card-ids init-query)snippet-ids (lib/native-query-snippet-ids init-query)](subquery-graph* metadata-providerablediff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.cljindex 8fb41473a4e..f1f5e80efa2 100644--- a/test/metabase/driver/sql/parameters/substitute_test.clj+++ b/test/metabase/driver/sql/parameters/substitute_test.clj@@ -386,6 +386,56 @@(substitute query {"snippet: outer" (lib/parsed-referenced-query-snippet-param 123 "{{snippet:symbol_is_A}}")"snippet: symbol_is_A" (lib/parsed-referenced-query-snippet-param 124 "symbol = 'A'")}))))))+(deftest ^:parallel substitute-repeated-native-query-snippets-test+ (testing "The same snippet can be used more than once, including from within other snippets"+ (let [query ["SELECT * FROM test_scores WHERE " (lib/parsed-param "snippet: outer")+ " OR " (lib/parsed-param "snippet: symbol_is_A")]]+ (is (=? ["SELECT * FROM test_scores WHERE (symbol = 'A' AND symbol = 'A') OR symbol = 'A'" nil]+ (substitute query {"snippet: outer" (lib/parsed-referenced-query-snippet-param+ 123 "({{snippet:symbol_is_A}} AND {{snippet:symbol_is_A}})")+ "snippet: symbol_is_A" (lib/parsed-referenced-query-snippet-param 124 "symbol = 'A'")}))))))++(deftest ^:parallel substitute-circular-native-query-snippets-test+ (testing "Snippets that (directly or indirectly) reference themselves are rejected"+ (let [query ["SELECT * FROM test_scores WHERE " (lib/parsed-param "snippet: a")]]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"circular"+ (substitute query {"snippet: a" (lib/parsed-referenced-query-snippet-param 1 "{{snippet: a}}")})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"circular"+ (substitute query {"snippet: a" (lib/parsed-referenced-query-snippet-param 1 "x {{snippet: b}}")+ "snippet: b" (lib/parsed-referenced-query-snippet-param 2 "y {{snippet: a}}")}))))))++(defn- doubling-snippets+ "Snippets `s0` ... `s<depth>` where each references the next one twice, and the last one has `leaf-content`."+ [depth leaf-content]+ (into {(str "snippet: s" depth) (lib/parsed-referenced-query-snippet-param (inc depth) leaf-content)}+ (for [i (range depth)+ :let [next-tag (str "{{snippet: s" (inc i) "}}")]]+ [(str "snippet: s" i) (lib/parsed-referenced-query-snippet-param (inc i) (str next-tag next-tag))])))++(deftest ^:parallel substitute-native-query-snippets-expansion-limit-test+ (testing "Snippets that expand into too many nested snippets are rejected"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"too many"+ (substitute ["SELECT " (lib/parsed-param "snippet: s0")]+ (doubling-snippets 20 ""))))))++(deftest ^:parallel substitute-native-query-snippets-length-limit-test+ (testing "Snippets that expand into a huge query are rejected"+ (let [leaf (str/join (repeat 10000 "x"))]+ (is (= (+ (count "SELECT ") (* 512 (count leaf)))+ (count (first (substitute ["SELECT " (lib/parsed-param "snippet: s0")]+ (doubling-snippets 9 leaf))))))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"too long"+ (substitute ["SELECT " (lib/parsed-param "snippet: s0")]+ (doubling-snippets 10 leaf)))))))+;;; ------------------------------------------ simple substitution — {{x}} ------------------------------------------(defn- substitute-e2e [sql params]diff --git a/test/metabase/query_processor/middleware/resolve_referenced_test.clj b/test/metabase/query_processor/middleware/resolve_referenced_test.cljindex ef9c75039f2..f69b374fad9 100644--- a/test/metabase/query_processor/middleware/resolve_referenced_test.clj+++ b/test/metabase/query_processor/middleware/resolve_referenced_test.clj@@ -366,3 +366,33 @@(testing "Should NOT throw an exception for valid non-cyclic chain";; This should complete, without throwing an exception, and return a dependency graph(is (some? (#'qp.resolve-referenced/check-for-circular-references entrypoint-query))))))))++(deftest shared-snippet-references-are-visited-once-test+ (testing "Snippets reachable through many paths are only walked once when checking for circular references"+ (let [depth 10+ snippet-id (fn [level which] (+ 100 (* 2 level) which))+ snippet-name (fn [level which] (str "snippet-" level "-" which))+ snippets (for [level (range (inc depth))+ which [0 1]+ :let [refs (when (< level depth)+ [[(snippet-id (inc level) 0) (snippet-name (inc level) 0)]+ [(snippet-id (inc level) 1) (snippet-name (inc level) 1)]])]]+ (make-snippet {:id (snippet-id level which)+ :name (snippet-name level which)+ :content (apply str "x" (for [[_ ref-name] refs]+ (str " {{snippet: " ref-name "}}")))+ :snippet-refs refs}))+ metadata-provider (lib.tu/mock-metadata-provider meta/metadata-provider {:native-query-snippets snippets})+ query (lib/query+ metadata-provider+ {:database (meta/id)+ :type :native+ :native {:query "SELECT {{snippet: snippet-0-0}}"+ :template-tags (make-snippet-template-tag (snippet-id 0 0) (snippet-name 0 0))}})+ fetches (atom 0)+ orig lib.metadata/native-query-snippet]+ (mt/with-dynamic-fn-redefs [lib.metadata/native-query-snippet (fn [mp id]+ (swap! fetches inc)+ (orig mp id))]+ (#'qp.resolve-referenced/check-for-circular-references query))+ (is (< @fetches (* 2 (count snippets)))))))