The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix slow `metabase.core.modules-test/modules-config-up-to-date-test` (#65168)
Cam Saulfd476e892b43
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex 9528376ae6a..c0a557f1e5e 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -1,6 +1,5 @@(ns dev.deps-graph(:require- [clojure.core.memoize :as memoize][clojure.edn :as edn][clojure.java.io :as io][clojure.string :as str]@@ -126,6 +125,7 @@(= (z/tag zloc) :uneval)))(mu/defn- find-requires :- [:maybe [:sequential ::zloc]]+ "Find all the zipper locations for `(require ...)` nodes."[zloc :- ::zloc](concat(when-not (comment-loc? zloc)@@ -261,41 +261,46 @@(file-dependencies "src/metabase/query_processor/middleware/permissions.clj"))-(def ^{:arglists '([])} dependencies- (memoize/ttl- (fn []- (doall (pmap file-dependencies (find-source-files))))- ;; memoize for five seconds- :ttl/threshold 5000))+(defn dependencies []+ (pmap file-dependencies (find-source-files)))(defn external-usages"All usages of a module named by `module-symb` outside that module."- [module-symb]- (for [dep (dependencies)- :when (not= (:module dep) module-symb)- ns-dep (:deps dep)- :when (= (:module ns-dep) module-symb)]- {:namespace (:namespace dep)- :module (:module dep)- :depends-on-namespace (:namespace ns-dep)- :depends-on-module (:module ns-dep)}))+ ([module-symb]+ (external-usages (dependencies) module-symb))++ ([deps module-symb]+ (for [dep deps+ :when (not= (:module dep) module-symb)+ ns-dep (:deps dep)+ :when (= (:module ns-dep) module-symb)]+ {:namespace (:namespace dep)+ :module (:module dep)+ :depends-on-namespace (:namespace ns-dep)+ :depends-on-module (:module ns-dep)})))(defn external-usages-by-namespace"Return a map of module namespace => set of external namespaces using it"- [module-symb]- (into (sorted-map)- (map (fn [[k v]]- [k (into (sorted-set) (map :namespace) v)]))- (group-by :depends-on-namespace (external-usages module-symb))))+ ([module-symb]+ (external-usages-by-namespace (dependencies) module-symb))++ ([deps module-symb]+ (into (sorted-map)+ (map (fn [[k v]]+ [k (into (sorted-set) (map :namespace) v)]))+ (group-by :depends-on-namespace (external-usages deps module-symb)))))(defn externally-used-namespaces"All namespaces from a module that are used outside that module."- [module-symb]- (into (sorted-set) (map :depends-on-namespace) (external-usages module-symb)))+ ([module-symb]+ (externally-used-namespaces (dependencies) module-symb))++ ([deps module-symb]+ (into (sorted-set) (map :depends-on-namespace) (external-usages deps module-symb))))(defn module-dependencies"Build a graph of module => set of modules it refers to."- ([]+ ([deps](letfn [(reduce-module-deps [module-deps module deps](reduce(fn [module-deps {dep-module :module, :as _dep}]@@ -305,15 +310,15 @@deps))(reduce-deps [module->deps {:keys [module deps]}](update module->deps module reduce-module-deps module deps))]- (reduce reduce-deps (sorted-map) (dependencies))))+ (reduce reduce-deps (sorted-map) deps)))- ([module]- (get (module-dependencies) module)))+ ([deps module]+ (get (module-dependencies deps) module)))(defn circular-dependencies"Build a graph of module => set of modules it refers to that also refer to this module."- ([]- (let [module->deps (module-dependencies)]+ ([deps]+ (let [module->deps (module-dependencies deps)](letfn [(circular-dependency? [module-x module-y](and (contains? (get module->deps module-x) module-y)(contains? (get module->deps module-y) module-x)))@@ -329,37 +334,40 @@[module circular-deps])))(keys module->deps)))))- ([module]- (get (circular-dependencies) module)))+ ([deps module]+ (get (circular-dependencies deps) module)))(defn non-circular-module-dependencies"A graph of [[module-dependencies]], but with modules that have any circular dependencies filtered out. This is mostlymeant to make it easier to fill out the `:metabase/modules` `:uses` section of the Kondo config, or to figure outwhich ones can easily get a consolidated API namespace without drama."- []- (let [circular-dependencies (circular-dependencies)]+ [deps]+ (let [circular-dependencies (circular-dependencies deps)](into (sorted-map)(remove (fn [[module _deps]](contains? circular-dependencies module)))- (module-dependencies))))+ (module-dependencies deps))))(defn module-usages-of-other-module"Information about how `module-x` uses `module-y`."- [module-x module-y]- (let [module-x-ns->module-y-ns (->> (external-usages module-y)- (filter #(= (:module %) module-x))- (map (juxt :namespace :depends-on-namespace)))]- (reduce- (fn [m [module-x-ns module-y-ns]]- (update m module-x-ns (fn [deps]- (conj (or deps (sorted-set)) module-y-ns))))- (sorted-map)- module-x-ns->module-y-ns)))+ ([module-x module-y]+ (module-usages-of-other-module (dependencies) module-x module-y))++ ([deps module-x module-y]+ (let [module-x-ns->module-y-ns (->> (external-usages deps module-y)+ (filter #(= (:module %) module-x))+ (map (juxt :namespace :depends-on-namespace)))]+ (reduce+ (fn [m [module-x-ns module-y-ns]]+ (update m module-x-ns (fn [deps]+ (conj (or deps (sorted-set)) module-y-ns))))+ (sorted-map)+ module-x-ns->module-y-ns))))(defn full-dependencies"Like [[dependencies]] but also includes transient dependencies."- []- (let [deps-graph (module-dependencies)+ [deps]+ (let [deps-graph (module-dependencies deps)expand-deps (fn expand-deps [deps](let [deps' (into (sorted-set)(mapcat deps-graph)@@ -372,33 +380,23 @@[k (expand-deps v)]))deps-graph)))-(defn module-deps-count []+(defn module-deps-count [deps](into (sorted-map)(map (fn [[k v]][k (count v)]))- (full-dependencies)))--(defn module-dependencies-mermaid []- (println "flowchart TD")- (doseq [[module deps] (module-dependencies)- dep deps]- (printf "%s-->%s\n" module dep)))--(defn module-dependencies-graphviz []- (println "digraph {")- (doseq [[module deps] (module-dependencies)- dep deps]- (printf " \"%s\" -> \"%s\"\n" module dep))- (println "}"))+ (full-dependencies deps)))(defn generate-config"Generate the Kondo config that should go in `.clj-kondo/config/modules/config.edn`."- []- (into (sorted-map)- (map (fn [[module uses]]- [module {:api (externally-used-namespaces module)- :uses uses}]))- (module-dependencies)))+ ([]+ (generate-config (dependencies)))++ ([deps]+ (into (sorted-map)+ (map (fn [[module uses]]+ [module {:api (externally-used-namespaces deps module)+ :uses uses}]))+ (module-dependencies deps))))(defn kondo-config"Read out the Kondo config for the modules linter."@@ -422,13 +420,16 @@diff))(defn kondo-config-diff- []- (-> (ddiff/diff- (update-vals (kondo-config) #(dissoc % :team))- (generate-config))- ddiff/minimize- kondo-config-diff-ignore-any- ddiff/minimize))+ ([]+ (kondo-config-diff (dependencies)))++ ([deps]+ (-> (ddiff/diff+ (update-vals (kondo-config) #(dissoc % :team))+ (generate-config deps))+ ddiff/minimize+ kondo-config-diff-ignore-any+ ddiff/minimize)))(defn print-kondo-config-diff"Print the diff between how the config would look if regenerated with [[generate-config]] versus how it looks in@@ -437,6 +438,6 @@(ddiff/pretty-print (kondo-config-diff)))(comment- (module-dependencies 'lib)+ (module-dependencies (dependencies) 'lib)(module-usages-of-other-module 'lib 'models))diff --git a/test/metabase/core/modules_test.clj b/test/metabase/core/modules_test.cljindex 4a482d43597..89c85b4b925 100644--- a/test/metabase/core/modules_test.clj+++ b/test/metabase/core/modules_test.clj@@ -140,7 +140,8 @@(deftest ^:parallel modules-config-up-to-date-test(testing (str "Please update .clj-kondo/config/modules/config.edn 🥰\n""[Pro Tip: use (dev.deps-graph/print-kondo-config-diff) to see the changes you need to make in a nicer format]\n")- (let [expected (dev.deps-graph/generate-config)+ (let [deps (dev.deps-graph/dependencies)+ expected (dev.deps-graph/generate-config deps)actual (dev.deps-graph/kondo-config)modules (set/union (set (keys expected))(set (keys actual)))]@@ -161,9 +162,9 @@:uses (reduce(partial merge-with set/union){}- (map #(dev.deps-graph/module-usages-of-other-module module %)+ (map #(dev.deps-graph/module-usages-of-other-module deps module %)missing))- :api (select-keys (dev.deps-graph/external-usages-by-namespace module) missing))))+ :api (select-keys (dev.deps-graph/external-usages-by-namespace deps module) missing))))(is (empty? missing)))(testing (format "Remove %s from %s" (pr-str extraneous) (pr-str ks))(is (empty? extraneous)))))))