Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[test] Optimize modules-test (#77065)

Oleksandr Yakushev5918cc9a6b3a
diff --git a/deps.edn b/deps.edn
index f289feeb998..04c058aef04 100644
--- a/deps.edn
+++ b/deps.edn
@@ -290,7 +290,7 @@
reifyhealth/specmonstah {:mvn/version "2.1.0"
:exclusions [org.clojure/clojure
org.clojure/clojurescript]} ; lets you write test fixtures that are clear, concise, and easy to maintain (clojure.spec)
- rewrite-clj/rewrite-clj {:mvn/version "1.2.50"} ; clojure source parsing and rewriting
+ rewrite-clj/rewrite-clj {:mvn/version "1.2.54"} ; clojure source parsing and rewriting
ring/ring-devel {:mvn/version "1.15.3"} ; reload clojure files on the fly
ring/ring-mock {:mvn/version "0.6.2"} ; creating Ring request maps for testing purposes
talltale/talltale {:mvn/version "0.5.14"}} ; generates fake data, useful for prototyping or load testing
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.clj
index 2571fa508cd..aa21a167416 100644
--- a/dev/src/dev/deps_graph.clj
+++ b/dev/src/dev/deps_graph.clj
@@ -18,6 +18,38 @@
(set! *warn-on-reflection* true)
+;; Many functions in this namespace re-parse the same files over and over again during testing, so introduce a
+;; mechanism for bounded caching of those parsed files.
+(def ^:dynamic *parsed-file-cache* nil)
+
+(defn- parse-file-all
+ "Calls `rewrite-.clj.parser/parse-file-all`, but first checks in `*parsed-file-cache*` if it is bound."
+ [file]
+ (if *parsed-file-cache*
+ (or (@*parsed-file-cache* file)
+ (get (swap! *parsed-file-cache* assoc file (r.parser/parse-file-all file)) file))
+ (r.parser/parse-file-all file)))
+
+(defn- skip-node? [node]
+ (let [tag (n/tag node)]
+ (or (#{:uneval :comment :newline :whitespace} tag)
+ (and (= tag :list)
+ (when-let [fst (first (n/children node))]
+ (and (= (n/tag fst) :token)
+ (= (:value fst) 'comment)))))))
+
+(defn- walk-parsed-ignore-comments!
+ "Simple fast recursive walker over `tree` which should be a file parsed with rewrite-clj. The passed consumer function
+ `f` should use side-effects to accumulate the results."
+ [f tree]
+ (letfn [(walk [node]
+ (when-not (skip-node? node)
+ ;; Don't descend into comments
+ (f node)
+ (when-let [children (:children node)]
+ (run! walk children))))]
+ (walk tree)))
+
(mu/defn- project-root-directory :- (ms/InstanceOfClass java.io.File)
^java.io.File []
(.. (java.nio.file.Paths/get (.toURI (io/resource "dev/deps_graph.clj")))
@@ -70,81 +102,36 @@
requiring-resolve
clojure.core/requiring-resolve})
-(mr/def ::node
- [:and
- :map
- [:fn
- {:error/message "valid rewrite-clj node"}
- #(not= (n/tag %) :unknown)]])
-
-(mr/def ::zloc
- [:tuple
- ::node
- :map])
-
-(mu/defn- require-loc?
- "Whether this zipper location points to a `(require ...)` node or a something similar (`classloader/require` or
- `requiring-resolve`)."
- [zloc :- ::zloc]
- (when (= (z/tag zloc) :list)
- (let [first-child (z/down zloc)]
- (when (= (z/tag first-child) :token)
- ;; Check all child symbols on the line since `require` might be called in a threading macro
- ;; like (-> 'ns require)
- (some require-symbols (z/child-sexprs zloc))))))
-
-(mu/defn- find-required-namespace :- [:maybe simple-symbol?]
- "Given a `zloc` pointing to one of the children of something like `(require ...)` find a required namespace symbol."
- [zloc :- ::zloc]
- (when-let [symbol-loc (z/find-depth-first zloc #(and (= (z/tag %) :token)
- (symbol? (z/sexpr %))
- (not= (z/sexpr %) 'quote)))]
- (let [symb (z/sexpr symbol-loc)]
- (if (qualified-symbol? symb)
- (symbol (namespace symb))
- symb))))
-
-(mu/defn- find-required-namespaces :- [:set simple-symbol?]
- "Given a zipper location pointing to a `(require ...)` node, find all the symbols it loads."
- [require-loc :- ::zloc]
- (loop [acc #{}, zloc (-> require-loc
- z/down ; require
- z/right)] ; second child
- (if-not zloc
- acc
- (recur (let [required-symbol (find-required-namespace zloc)]
- (cond-> acc
- required-symbol (conj required-symbol)))
- (z/right zloc)))))
-
-(mu/defn- comment-loc?
- [zloc :- ::zloc]
- (or (and (= (z/tag zloc) :list)
- (let [child-loc (z/down zloc)]
- (and (= (z/tag child-loc) :token)
- (= (z/sexpr child-loc) 'comment))))
- (= (z/tag zloc) :uneval)))
-
-(mu/defn- find-requires :- [:maybe [:sequential ::zloc]]
- "Find all the zipper locations for `(require ...)` nodes."
- [zloc :- ::zloc]
- (concat
- (when-not (comment-loc? zloc)
- (if (require-loc? zloc)
- [zloc]
- (when-let [down (z/down zloc)]
- (find-requires down))))
- (when-let [right (z/right zloc)]
- (find-requires right))))
+(defn- find-required-namespaces
+ "Find all `(require ...)` forms in a file and return symbols it they load."
+ [file]
+ (let [acc (atom #{})]
+ (walk-parsed-ignore-comments!
+ (fn [node]
+ (when (= (n/tag node) :list)
+ (when-let [[first-child & other :as children] (remove skip-node? (n/children node))]
+ (when (and first-child
+ (= (n/tag first-child) :token)
+ ;; Check all child symbols on the line since `require` might be called in a threading macro
+ ;; like (-> 'ns require)
+ (some #(and (= (n/tag %) :token)
+ (require-symbols (n/sexpr %)))
+ children))
+ (run! #(when (and (= (n/tag %) :quote))
+ (when-let [in (some-> (n/children %) first n/sexpr)]
+ (when (symbol? in)
+ (swap! acc conj (if (qualified-symbol? in)
+ (symbol (namespace in))
+ in)))))
+ other)))))
+ (parse-file-all file))
+ @acc))
(mu/defn- find-dynamically-loaded-namespaces :- [:set simple-symbol?]
"Find the set of namespace symbols for namespaces loaded by `require` and friends in a `file`."
[file]
(try
- (let [node (r.parser/parse-file-all file)
- zloc (z/of-node node)
- requires (find-requires zloc)]
- (into #{} (mapcat find-required-namespaces) requires))
+ (find-required-namespaces file)
(catch Throwable e
(throw (ex-info (format "Error in file %s: %s" (str file) (ex-message e))
{:file file}
@@ -176,7 +163,7 @@
(some->> x z/of-node z/down z/right z/right z/right z/sexpr))]
(swap! defentz conj target-ns))
x)
- (z/of-node (r.parser/parse-file-all file)))
+ (z/of-node (parse-file-all file)))
@defentz)))
(mu/defn find-defenterprise-schemas
@@ -194,7 +181,7 @@
(some->> x z/of-node z/down z/right z/right z/right z/right z/right z/sexpr))]
(swap! defentz conj target-ns))
x)
- (z/of-node (r.parser/parse-file-all file)))
+ (z/of-node (parse-file-all file)))
@defentz)))
(comment
@@ -285,7 +272,7 @@
"Calculate information about all the modules dependencies for all *SOURCE* files in the Metabase project by parsing
the files."
[]
- (pmap file-dependencies (find-source-files)))
+ (map file-dependencies (find-source-files)))
(defn external-usages
"All usages of a module named by `module-symb` outside that module."
@@ -752,23 +739,15 @@
[file]
(try
(let [models (atom #{})]
- (loop [stack [(z/of-node (r.parser/parse-file-all file))]]
- (when-let [zloc (peek stack)]
- (let [stack' (pop stack)
- stack' (if-let [right (z/right zloc)]
- (conj stack' right)
- stack')]
- (if (comment-loc? zloc)
- (recur stack')
- (do
- (when (and (= (z/tag zloc) :token)
- (keyword? (z/sexpr zloc))
- (= "model" (namespace (z/sexpr zloc)))
- (Character/isUpperCase ^char (first (name (z/sexpr zloc)))))
- (swap! models conj (z/sexpr zloc)))
- (recur (if-let [child (z/down zloc)]
- (conj stack' child)
- stack')))))))
+ (walk-parsed-ignore-comments!
+ (fn [node]
+ (when (= (n/tag node) :token)
+ (let [sexpr (n/sexpr node)]
+ (when (and (keyword? sexpr)
+ (= "model" (namespace sexpr))
+ (Character/isUpperCase ^char (first (name sexpr))))
+ (swap! models conj sexpr)))))
+ (parse-file-all file))
@models)
(catch Throwable e
(throw (ex-info (format "Error scanning model keywords in %s: %s" (str file) (ex-message e))
@@ -779,31 +758,22 @@
"Find all models with their `t2/table-name` defined in this file."
[file]
(let [models (atom #{})]
- (loop [stack [(z/of-node (r.parser/parse-file-all file))]]
- (when-let [zloc (peek stack)]
- (let [stack' (pop stack)
- stack' (if-let [right (z/right zloc)]
- (conj stack' right)
- stack')]
- (if (comment-loc? zloc)
- (recur stack')
- (do
- (when (= (z/tag zloc) :list)
- (let [first-child (z/down zloc)]
- (when (and first-child
- (= (z/tag first-child) :token)
- (some-> (:string-value (z/node first-child)) (str/ends-with? "/defmethod")))
- (let [second-zloc (z/right first-child)
- third-zloc (some-> second-zloc z/right)]
- (when (and second-zloc
- (= (z/sexpr second-zloc) 't2/table-name)
- third-zloc
- (keyword? (z/sexpr third-zloc))
- (= "model" (namespace (z/sexpr third-zloc))))
- (swap! models conj (z/sexpr third-zloc)))))))
- (recur (if-let [child (z/down zloc)]
- (conj stack' child)
- stack')))))))
+ (walk-parsed-ignore-comments!
+ (fn [node]
+ (when (= (n/tag node) :list)
+ (let [[fst snd trd :as children] (remove skip-node? (n/children node))]
+ (when (and fst (= (n/tag fst) :token)
+ (some-> (n/string fst) (str/ends-with? "/defmethod")))
+ (when (and fst snd trd
+ (= (n/tag fst) :token)
+ (= (n/tag snd) :token)
+ (= (n/tag trd) :token)
+ (some-> (n/string fst) (str/ends-with? "/defmethod"))
+ (= (n/sexpr snd) 't2/table-name)
+ (keyword? (n/sexpr trd))
+ (= "model" (namespace (n/sexpr trd))))
+ (swap! models conj (n/sexpr trd)))))))
+ (parse-file-all file))
@models))
(defn model-ownership
@@ -885,37 +855,38 @@
([]
(model-boundary-violations (kondo-config)))
([kondo-config]
- (let [ownership (model-ownership)]
- (into []
- (comp
- (mapcat
- (fn [file]
- (try
- (let [ns-symb (-> (ns.file/read-file-ns-decl file)
- ns.parse/name-from-ns-decl)
- mod (module ns-symb)]
- (when (and mod
- (not (contains? model-boundary-exempt-namespaces ns-symb)))
- (let [model-imports (get-in kondo-config [mod :model-imports] #{})
- models (find-model-keywords file)
- rel-path (file->path-relative-to-project-root file)]
- (for [model models
- :let [defining-mod (get ownership model)]
- :when (not= defining-mod mod)
- :let [model-exports (when defining-mod
- (get-in kondo-config [defining-mod :model-exports] #{}))]
- violation-type (model-reference-violations
- model defining-mod model-exports model-imports)]
- {:file rel-path
- :module mod
- :model model
- :defining-module defining-mod
- :violation-type violation-type}))))
- (catch Throwable e
- (throw (ex-info (format "Error checking model boundaries in %s" (str file))
- {:file file}
- e)))))))
- (find-source-files)))))
+ (model-boundary-violations kondo-config (model-ownership)))
+ ([kondo-config ownership]
+ (into []
+ (comp
+ (mapcat
+ (fn [file]
+ (try
+ (let [ns-symb (-> (ns.file/read-file-ns-decl file)
+ ns.parse/name-from-ns-decl)
+ mod (module ns-symb)]
+ (when (and mod
+ (not (contains? model-boundary-exempt-namespaces ns-symb)))
+ (let [model-imports (get-in kondo-config [mod :model-imports] #{})
+ models (find-model-keywords file)
+ rel-path (file->path-relative-to-project-root file)]
+ (for [model models
+ :let [defining-mod (get ownership model)]
+ :when (not= defining-mod mod)
+ :let [model-exports (when defining-mod
+ (get-in kondo-config [defining-mod :model-exports] #{}))]
+ violation-type (model-reference-violations
+ model defining-mod model-exports model-imports)]
+ {:file rel-path
+ :module mod
+ :model model
+ :defining-module defining-mod
+ :violation-type violation-type}))))
+ (catch Throwable e
+ (throw (ex-info (format "Error checking model boundaries in %s" (str file))
+ {:file file}
+ e)))))))
+ (find-source-files))))
(comment
(model-ownership)
diff --git a/test/metabase/core/modules_test.clj b/test/metabase/core/modules_test.clj
index d4d14747875..1dcc1b449b9 100644
--- a/test/metabase/core/modules_test.clj
+++ b/test/metabase/core/modules_test.clj
@@ -14,6 +14,9 @@
(set! *warn-on-reflection* true)
+(use-fixtures :once #(binding [dev.deps-graph/*parsed-file-cache* (atom {})]
+ (%)))
+
(defn- modules-config
"Kondo modules config."
[]
@@ -32,7 +35,7 @@
(def ^:private teams-to-reassign #{"Admin Webapp" "DashViz"})
-(deftest ^:parallel all-modules-have-teams-test
+(deftest all-modules-have-teams-test
(testing "All modules should have a valid :team owner"
(let [teams (teams)]
(doseq [[module config] (modules-config)]
@@ -77,7 +80,7 @@
module-name])
module-names))
-(deftest ^:parallel modules-should-be-sorted-by-name-test
+(deftest modules-should-be-sorted-by-name-test
(testing "Modules configs should sorted by module name with enterprise/modules appearing last"
(let [actual (module-names-in-file-order)
expected (sort-module-names actual)]
@@ -98,7 +101,7 @@
(when-let [zloc' (z/right config-zloc)]
(recur zloc')))))
-(deftest ^:parallel module-api-namespaces-should-be-sorted-test
+(deftest module-api-namespaces-should-be-sorted-test
(testing "Module :api namespaces should be sorted"
(do-each-module-config
(fn [module config-zloc]
@@ -118,7 +121,7 @@
(is (= (sort api-namespaces)
api-namespaces))))))))
-(deftest ^:parallel module-uses-should-be-sorted-test
+(deftest module-uses-should-be-sorted-test
(testing "Module :uses namespaces should be sorted"
(do-each-module-config
(fn [module config-zloc]
@@ -138,7 +141,7 @@
(is (= (sort-module-names uses)
uses))))))))
-(deftest ^:parallel modules-config-up-to-date-test
+(deftest modules-config-up-to-date-test
(testing (str "Please update .clj-kondo/config/modules/config.edn 🥰\n"
"[Pro Tip: use (dev.deps-graph/print-kondo-config-diff) to see the changes you need to make in a nicer format]\n")
(let [deps (dev.deps-graph/dependencies)
@@ -173,7 +176,7 @@
(defn- rest-module? [module]
(str/ends-with? module "-rest"))
-(deftest ^:parallel do-not-use-rest-modules-in-other-modules-test
+(deftest do-not-use-rest-modules-in-other-modules-test
(doseq [[module {:keys [uses], :as _config}] (dev.deps-graph/kondo-config)
:when (not (rest-module? module))
used-module (when (set? uses)
@@ -187,7 +190,7 @@
;;;; Model boundary tests
-(deftest ^:parallel model-boundaries-test
+(deftest model-boundaries-test
(testing "Model boundary enforcement\n"
(let [ownership (dev.deps-graph/model-ownership)
known-models (set (keys ownership))
@@ -212,7 +215,7 @@
(testing (format "\n'%s' exports %s (owned by %s)" module model (get ownership model))
(is (= module (get ownership model)))))))))
-(deftest ^:parallel model-config-not-stale-test
+(deftest model-config-not-stale-test
(testing "Model exports and imports should not list models that are unused.\n"
(let [{computed-exports :model-exports
computed-imports :model-imports} (dev.model-boundary-config/compute-model-boundaries)
@@ -229,7 +232,7 @@
module direction config-key)
(is (empty? (sort stale))))))))
-(deftest ^:parallel model-exports-sorted-test
+(deftest model-exports-sorted-test
(testing "Module :model-exports should be sorted"
(do-each-module-config
(fn [module config-zloc]
@@ -245,7 +248,7 @@
(is (= (sort exports)
exports))))))))
-(deftest ^:parallel model-imports-sorted-test
+(deftest model-imports-sorted-test
(testing "Module :model-imports should be sorted"
(do-each-module-config
(fn [module config-zloc]