Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use a much larger maximum nesting limit while parsing YAML (#76528)

Braden Shepherdson78e968bd693d
diff --git a/src/metabase/util/yaml.clj b/src/metabase/util/yaml.clj
index c2a859a7cac..e7e68b792d1 100644
--- a/src/metabase/util/yaml.clj
+++ b/src/metabase/util/yaml.clj
@@ -9,17 +9,35 @@
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.files :as u.files]
- [metabase.util.log :as log]
- [potemkin :as p])
+ [metabase.util.log :as log])
(:import
(java.nio.file Files Path)
(java.time.temporal Temporal)))
(set! *warn-on-reflection* true)
-(p/import-vars
- [yaml
- parse-stream])
+(def ^:const nesting-depth-limit
+ "Maximum YAML nesting depth allowed when parsing.
+
+ SnakeYAML defaults to 50, which is too shallow for deeply nested MBQL. MBQL sometimes contains expressions with many
+ nested function calls like `replace(replace(...))`. There's no depth limit on the YAML output, so they are perfectly
+ valid queries that can be saved, executed and exported. But with the default depth limit, they cannot be imported,
+ leaving remote-sync users stuck (see #71257 / UXW-3770).
+
+ We raise the limit so that anything Metabase can export can also be re-imported, while keeping a finite bound as a
+ sanity guard against pathological/malicious input. Callers may override via the `:nesting-depth-limit` opt."
+ 10000)
+
+(defn- with-parse-defaults
+ "Merges Metabase's default parsing options into `opts`; preserving any values the caller supplied."
+ [opts]
+ (merge {:nesting-depth-limit nesting-depth-limit} opts))
+
+(defn parse-stream
+ "Returns Clojure data structures for a stream of YAML read from `reader`, with opts passed to
+ clj-yaml. Applies Metabase's default parsing options (see [[with-parse-defaults]])."
+ [reader & {:as opts}]
+ (yaml/parse-stream reader (with-parse-defaults opts)))
(defn- vectorized
"Returns x with lazy seqs converted to vectors wherever they appear in the data structure."
@@ -47,9 +65,10 @@
(yaml/generate-string x opts))
(defn parse-string
- "Returns a Clojure object parsed from YAML in string s with opts passed to clj-yaml."
+ "Returns a Clojure object parsed from YAML in string s with opts passed to clj-yaml.
+ Applies Metabase's default parsing options (see [[with-parse-defaults]])."
[s & {:as opts}]
- (vectorized (yaml/parse-string s opts)))
+ (vectorized (yaml/parse-string s (with-parse-defaults opts))))
;; Legacy API:
diff --git a/test/metabase/util/yaml_test.clj b/test/metabase/util/yaml_test.clj
new file mode 100644
index 00000000000..7a8d7ea7ece
--- /dev/null
+++ b/test/metabase/util/yaml_test.clj
@@ -0,0 +1,46 @@
+(ns metabase.util.yaml-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.util.yaml :as yaml])
+ (:import
+ (java.io File StringReader)))
+
+(set! *warn-on-reflection* true)
+
+(defn- nested-structure
+ "Builds a value nested `depth` levels deep, e.g. `(nested-structure 3)` => `[[[\"leaf\"]]]`.
+
+ This mimics the deeply nested YAML produced by serializing MBQL expressions with many nested
+ function calls (see #71257 / UXW-3770)."
+ [depth]
+ (-> (iterate vector "leaf")
+ (nth depth)))
+
+(deftest ^:parallel parse-string-allows-deep-nesting-test
+ (testing "parse-string allows nesting deeper than SnakeYAML's default limit of 50 (#71257)"
+ (let [deep (nested-structure 500)]
+ (is (= deep
+ (-> deep yaml/generate-string yaml/parse-string))
+ "deeply nested YAML that Metabase can generate must round-trip back through parse-string")))
+ (testing "callers may still tighten the nesting limit via :nesting-depth-limit"
+ (is (thrown-with-msg? Exception #"Nesting Depth exceeded"
+ (-> (nested-structure 500)
+ yaml/generate-string
+ (yaml/parse-string :nesting-depth-limit 10))))))
+
+(deftest ^:parallel from-file-allows-deep-nesting-test
+ (testing "from-file allows nesting deeper than SnakeYAML's default limit (this is the serdes import path)"
+ (let [deep (nested-structure 500)
+ ^File tmp (File/createTempFile "deep-yaml" ".yaml")]
+ (try
+ (spit tmp (yaml/generate-string deep))
+ (is (= deep (yaml/from-file tmp)))
+ (finally
+ (.delete tmp))))))
+
+(deftest ^:parallel parse-stream-allows-deep-nesting-test
+ (testing "parse-stream allows nesting deeper than SnakeYAML's default limit"
+ (let [deep (nested-structure 500)
+ s (yaml/generate-string deep)]
+ (with-open [r (StringReader. s)]
+ (is (= deep (yaml/parse-stream r)))))))