The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
New deps graph tooling to power Mage test watching (#66345)
Cam Saul7f68894e80c4
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex e59234160d6..924ccc00f62 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -204,6 +204,21 @@;; has 2 defenterprise-schemas, both to the same ns:(find-defenterprise-schemas "src/metabase/sso/ldap/default_implementation.clj"))+(def ^:private ^String project-root (System/getProperty "user.dir"))++(defn- file->path-relative-to-project-root+ "Get the path of a file relative to the project (repo) root directory e.g.++ (file->path-relative-to-project-root \"/home/cam/metabase/deps.edn\")+ ;; =>+ \"deps.edn\""+ [file]+ (let [file (io/file file)+ path (.getAbsolutePath file)]+ (if (str/starts-with? path project-root)+ (subs path (inc (count project-root))) ; project root won't include trailing `/`+ path)))+(def ^:private ignored-dependencies"Technically `config` 'uses' `enterprise/core` and `test` since it tries to load them to see if they exist so we knowif EE/test code is available; however we can ignore them since they're not 'real' usages. So add them here so we@@ -212,13 +227,16 @@(mu/defn- file-dependencies :- [:map[:namespace simple-symbol?]+ [:filename string?] ; filename is relative to [[project-root]][:module symbol?][:deps [:sequential[:map[:namespace simple-symbol?][:module symbol?][:dynamic {:optional true} :keyword]]]]]- [file]+ [file :- [:or+ string?+ [:fn {:error/message "Instance of a java.io.File"} #(instance? java.io.File %)]]](try(let [decl (ns.file/read-file-ns-decl file)ns-symb (ns.parse/name-from-ns-decl decl)@@ -239,6 +257,7 @@#_defenterprise-deps#_defenterprise-schema-deps])]{:namespace ns-symb+ :filename (file->path-relative-to-project-root file):module (module ns-symb):deps (sort-by pr-str(keep (fn [required-ns]@@ -262,7 +281,10 @@(file-dependencies "src/metabase/query_processor/middleware/permissions.clj"))-(defn dependencies []+(defn dependencies+ "Calculate information about all the modules dependencies for all *SOURCE* files in the Metabase project by parsing+ the files."+ [](pmap file-dependencies (find-source-files)))(defn external-usages@@ -518,14 +540,15 @@(defn leaf-modules"Modules that are leaf nodes in the module dependency tree -- nothing else depends on them."- []- (let [deps (dependencies)]- (into (sorted-set)- (comp (map :module)- (keep (fn [module]- (when (zero? (count (external-usages deps module)))- module))))- deps)))+ ([]+ (leaf-modules (dependencies)))+ ([deps]+ (into (sorted-set)+ (comp (map :module)+ (keep (fn [module]+ (when (zero? (count (external-usages deps module)))+ module))))+ deps)))(defn non-dependencies"Modules that `module` does not depend on, either directly or indirectly -- changes to any of these modules should not@@ -534,6 +557,7 @@(let [deps (dependencies)all-modules (into (sorted-set) (map :module) deps)module-deps (set (keys (all-module-deps-paths deps module)))]+ #_{:clj-kondo/ignore [:discouraged-var]}(printf "Module %s depends on %d/%d (%.1f%%) other modules.\n"module(count module-deps)@@ -575,3 +599,129 @@new-deps (simulate-rename deps old-namespace->new-namespace)new-module-deps (into (sorted-set) (keys (all-module-deps-paths new-deps module)))](set/difference old-module-deps new-module-deps)))++(mu/defn- module->source-files :- [:set :string]+ "Return the set of all *source* filenames (relative to the [[project-root]] directory) for a `module`."+ [deps module]+ (into+ (sorted-set)+ (comp (filter #(= (:module %) module))+ (map :filename))+ deps))++(defn- file->namespace+ "Infer the Clojure namespace from a core project filename.++ (file->namespace \"/home/cam/metabase/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj\")+ ;; =>+ metabase-enterprise.advanced-permissions.common"+ [file]+ (-> file+ file->path-relative-to-project-root+ (str/replace #"^enterprise/backend/" "")+ (str/replace #"^(?:(?:src)|(?:test))/" "")+ (str/replace #"\.clj[cs]?$" "")+ (str/replace #"/" ".")+ (str/replace #"_" "-")+ symbol))++(defn- module->all-deps [deps module]+ (keys (all-module-deps-paths deps module)))++(defn test-filenames->relevant-source-filenames+ "Given a collection of `test-filenames`, return the set of source filenames (relative to the project root directory)+ that when changed should trigger these tests."+ ([test-filenames]+ (test-filenames->relevant-source-filenames (dependencies) test-filenames))+ ([deps test-filenames]+ (into+ (sorted-set)+ (comp (map file->namespace)+ (map module)+ (distinct)+ (mapcat (fn [module]+ (into #{module} (module->all-deps deps module))))+ (distinct)+ (mapcat #(module->source-files deps %)))+ test-filenames)))++(comment+ ;; should include source files for `settings-rest` itself as well as all modules used either directly or indirectly+ ;; by `settings-rest` (currently this set is huge, 955 files as of 2025-11-26)+ (test-filenames->relevant-source-filenames ["test/metabase/settings_rest/api_test.clj"]))++(defn- direct-dependents+ "Set of modules that directly depend on `module`."+ [deps module]+ (into (sorted-set)+ (keep (fn [ns-info]+ (when (some (fn [ns-deps]+ (= (:module ns-deps) module))+ (:deps ns-info))+ (:module ns-info))))+ deps))++(comment+ (direct-dependents (dependencies) 'driver)+ (direct-dependents (dependencies) 'settings-rest))++(defn- indirect-dependents+ "Set of modules that either directly or indirectly depend on `module`."+ ([deps module]+ (indirect-dependents deps module (sorted-set)))+ ([deps module acc]+ (let [module-deps (direct-dependents deps module)+ new-deps (set/difference module-deps acc)+ acc (into acc new-deps)]+ (reduce+ (fn [acc new-dep]+ (indirect-dependents deps new-dep acc))+ acc+ new-deps))))++(comment+ (indirect-dependents (dependencies) 'settings-rest))++(defn- module->dependents [deps module]+ (into #{module}+ (indirect-dependents deps module)))++(comment+ (module->dependents (dependencies) 'settings-rest))++(defn- module->test-directory [module]+ (let [parent-dir (case (namespace module)+ nil "test/metabase/"+ "enterprise" "enterprise/backend/test/metabase_enterprise/")+ module-dir (str/replace (name module) #"-" "_")]+ (str parent-dir module-dir)))++(mu/defn- module->test-files :- [:set :string]+ "Return the set of test filenames associated with a `module`."+ [module :- :symbol]+ (let [test-dir (module->test-directory module)+ test-filenames (ns.find/find-sources-in-dir (io/file test-dir))]+ (into+ (sorted-set)+ (map file->path-relative-to-project-root)+ test-filenames)))++(defn source-filenames->relevant-test-filenames+ "Given a collection of `source-filenames`, return the set of test filenames (relative to the project root directory)+ that we should re-run when any of `source-filenames` change."+ ([source-filenames]+ (source-filenames->relevant-test-filenames (dependencies) source-filenames))+ ([deps source-filenames]+ (into+ (sorted-set)+ (comp (map file->namespace)+ (map module)+ (distinct)+ (mapcat #(module->dependents deps %))+ (mapcat module->test-files))+ source-filenames)))++(comment+ ;; should only include tests for `settings-rest`, `api-routes`, `core`, and the handful of random modules that use+ ;; `settings-rest` (21 files as of 2025-11-26)+ (source-filenames->relevant-test-filenames ["src/metabase/settings_rest/api.clj"]))