The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Measure module-boundary metrics at any commit (#84121)
Chris Truter7999d3c9ecbc
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex d6bfcc8b269..d2df258158b 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -9,6 +9,7 @@[clojure.tools.namespace.parse :as ns.parse][clojure.walk :as walk][dev.module-graph :as module-graph]+ [dev.module-source :as module-source][hooks.common.modules :as modules][lambdaisland.deep-diff2 :as ddiff][metabase.util :as u]@@ -39,26 +40,6 @@(get (swap! *parsed-file-cache* assoc file (r.parser/parse-file-all file)) file))(r.parser/parse-file-all file)))-(defn- skip-node? [node]- (let [tag (n/tag node)]- (or (#{:uneval :comment :newline :whitespace} tag)- (and (= tag :list)- (when-let [fst (first (n/children node))]- (and (= (n/tag fst) :token)- (= (:value fst) 'comment)))))))--(defn- walk-parsed-ignore-comments!- "Simple fast recursive walker over `tree` which should be a file parsed with rewrite-clj. The passed consumer function- `f` should use side-effects to accumulate the results."- [f tree]- (letfn [(walk [node]- (when-not (skip-node? node)- ;; Don't descend into comments- (f node)- (when-let [children (:children node)]- (run! walk children))))]- (walk tree)))-(mu/defn- project-root-directory :- (ms/InstanceOfClass java.io.File)^java.io.File [](.. (java.nio.file.Paths/get (.toURI (io/resource "dev/deps_graph.clj")))@@ -94,39 +75,10 @@(mapcat ns.find/find-sources-in-dir(list* (source-root) (enterprise-source-root) (plugin-source-roots))))-(def ^:private require-symbols- '#{require- clojure.core/require- classloader/require- metabase.classloader.core/require- metabase.classloader.impl/require- requiring-resolve- clojure.core/requiring-resolve})-(defn- find-required-namespaces"Find all `(require ...)` forms in a file and return symbols it they load."[file]- (let [acc (atom #{})]- (walk-parsed-ignore-comments!- (fn [node]- (when (= (n/tag node) :list)- (when-let [[first-child & other :as children] (remove skip-node? (n/children node))]- (when (and first-child- (= (n/tag first-child) :token)- ;; Check all child symbols on the line since `require` might be called in a threading macro- ;; like (-> 'ns require)- (some #(and (= (n/tag %) :token)- (require-symbols (n/sexpr %)))- children))- (run! #(when (and (= (n/tag %) :quote))- (when-let [in (some-> (n/children %) first n/sexpr)]- (when (symbol? in)- (swap! acc conj (if (qualified-symbol? in)- (symbol (namespace in))- in)))))- other)))))- (parse-file-all file))- @acc))+ (module-source/dynamic-requires (parse-file-all file)))(mu/defn- find-dynamically-loaded-namespaces :- [:set simple-symbol?]"Find the set of namespace symbols for namespaces loaded by `require` and friends in a `file`."@@ -211,7 +163,9 @@"Technically `config` 'uses' `enterprise/core` and `test` since it tries to load them to see if they exist so we knowif EE/test code is available; however we can ignore them since they're not 'real' usages. So add them here so wedon't include them in our deps tree."- '{metabase.config.core #{metabase-enterprise.core.dummy-namespace metabase.test.dummy-namespace}})+ ;; Older commits, which [[dev.module-history]] measures, name this namespace `metabase.config`.+ '{metabase.config #{metabase-enterprise.core.dummy-namespace metabase.test.dummy-namespace}+ metabase.config.core #{metabase-enterprise.core.dummy-namespace metabase.test.dummy-namespace}})(def ^:private ModuleConfig[:map {:closed true}@@ -241,41 +195,51 @@[:module symbol?][:dynamic {:optional true} :keyword]]]]])+(def ^:private SourceRequires+ [:map {:closed true}+ [:namespace [:maybe simple-symbol?]]+ [:static [:set simple-symbol?]]+ [:dynamic [:set simple-symbol?]]])++(mu/defn resolve-requires :- FileDependencies+ "The dependency row for one source file at `filename`, resolving its namespace and each namespace it requires to a+ module through `prefix->module`.+ The third argument has the shape [[module-source/source-requires]] returns.+ A namespace required both statically and dynamically counts as static."+ [prefix->module :- PrefixToModule+ filename :- :string+ {ns-symb :namespace, :keys [static dynamic]} :- SourceRequires]+ {:namespace ns-symb+ :filename filename+ :module (modules/resolve-module prefix->module ns-symb)+ :deps (sort-by pr-str+ (keep (fn [required-ns]+ (when-let [module (modules/resolve-module prefix->module required-ns)]+ (when-not (some-> ignored-dependencies ns-symb required-ns)+ (merge+ {:namespace required-ns+ :module module}+ (when-not (contains? static required-ns)+ {:dynamic :require-and-friends})))))+ (into (sorted-set) cat [static dynamic])))})+(mu/defn- file-dependencies :- FileDependencies[prefix->module :- PrefixToModulefile :- File](try- (let [decl (ns.file/read-file-ns-decl file)- ns-symb (ns.parse/name-from-ns-decl decl)- static-deps (ns.parse/deps-from-ns-decl decl)- dynamic-deps (for [symb (find-dynamically-loaded-namespaces file)]- (vary-meta symb assoc ::dynamic :require-and-friends))- ;;- ;; excluded from the diff for now, see https://metaboat.slack.com/archives/C0669P4AF9N/p1745875106092029 for- ;; rationale.- ;;- ;; defenterprise-deps (for [symb (find-defenterprises file)]- ;; (vary-meta symb assoc ::dynamic :defenterprise))- ;; defenterprise-schema-deps (for [symb (find-defenterprise-schemas file)]- ;; (vary-meta symb assoc ::dynamic :defenterprise-schema))- deps (into (sorted-set) cat- [static-deps- dynamic-deps- #_defenterprise-deps- #_defenterprise-schema-deps])]- {:namespace ns-symb- :filename (file->path-relative-to-project-root file)- :module (modules/resolve-module prefix->module ns-symb)- :deps (sort-by pr-str- (keep (fn [required-ns]- (when-let [module (modules/resolve-module prefix->module required-ns)]- (when-not (some-> ignored-dependencies ns-symb required-ns)- (merge- {:namespace required-ns- :module module}- (when-let [dynamic-type (::dynamic (meta required-ns))]- {:dynamic dynamic-type})))))- deps))})+ (let [decl (ns.file/read-file-ns-decl file)]+ ;; excluded from the diff for now, see https://metaboat.slack.com/archives/C0669P4AF9N/p1745875106092029 for+ ;; rationale.+ ;;+ ;; defenterprise-deps (for [symb (find-defenterprises file)]+ ;; (vary-meta symb assoc ::dynamic :defenterprise))+ ;; defenterprise-schema-deps (for [symb (find-defenterprise-schemas file)]+ ;; (vary-meta symb assoc ::dynamic :defenterprise-schema))+ (resolve-requires prefix->module+ (file->path-relative-to-project-root file)+ {:namespace (ns.parse/name-from-ns-decl decl)+ :static (set (ns.parse/deps-from-ns-decl decl))+ :dynamic (find-dynamically-loaded-namespaces file)}))(catch Throwable e(throw (ex-info (format "Error calculating dependencies for %s" file){:file file}@@ -796,7 +760,7 @@[file :- File](try(let [models (atom #{})]- (walk-parsed-ignore-comments!+ (module-source/walk-parsed-ignore-comments!(fn [node](when (= (n/tag node) :token)(let [sexpr (n/sexpr node)]@@ -815,10 +779,10 @@"Find all models with their `t2/table-name` defined in this file."[file :- File](let [models (atom #{})]- (walk-parsed-ignore-comments!+ (module-source/walk-parsed-ignore-comments!(fn [node](when (= (n/tag node) :list)- (let [[fst snd trd :as children] (remove skip-node? (n/children node))]+ (let [[fst snd trd] (remove module-source/skip-node? (n/children node))](when (and fst (= (n/tag fst) :token)(some-> (n/string fst) (str/ends-with? "/defmethod")))(when (and fst snd trd@@ -1024,13 +988,15 @@These values are not ratcheted because any source change can move them. Use namespace sizes to trackcycle reduction: splitting a module can grow a cycle's module count without removing namespaces.Density compares how tangled cycles of different sizes are.- Cycle rank is an upper bound on the edges to cut."+ Cycle rank is an upper bound on the edges to cut.+ Pass `cycles` from [[module-cycles]] for the same `deps` to reuse them."([](module-boundary-stats (dependencies) (kondo-config)))([deps config]+ (module-boundary-stats deps config (module-cycles deps)))+ ([deps config cycles](let [any-modules (into #{} (keep (fn [[module cfg]] (when (= :any (:api cfg)) module))) config)ns-counts (frequencies (keep :module deps))- cycles (module-cycles deps)upward (upward-references* deps config cycles)]{:api-any-namespaces (count (filter #(contains? any-modules (:module %)) deps)):module-count (count config)diff --git a/dev/src/dev/module_history.clj b/dev/src/dev/module_history.cljnew file mode 100644index 00000000000..ae700c7ffe8--- /dev/null+++ b/dev/src/dev/module_history.clj@@ -0,0 +1,667 @@+(ns dev.module-history+ "Module-boundary metrics at any commit, read from git without a checkout.++ Every commit is measured with the code in this tree: its sources and module config come from git, and today's+ [[dev.deps-graph]] resolves and measures them, so every data point follows the same rules.++ Results are cached outside the repo, under `$MB_MODULE_HISTORY_CACHE` or `~/.cache/metabase/module-history`,+ so worktrees and runs share them.+ Parsed files are keyed by git blob and a hash of [[dev.module-source]]; per-commit metrics are keyed by+ commit and a hash of every namespace that computes them.+ Changing that code starts a fresh cache layer instead of serving stale results.++ (backfill! {:since \"2025-10-01\", :monthly-since \"2023-12-01\", :out \"history.csv\"})"+ (:require+ [clojure.data.csv :as csv]+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.java.shell :as sh]+ [clojure.string :as str]+ [dev.deps-graph :as deps-graph]+ [dev.module-graph :as module-graph]+ [dev.module-source :as module-source]+ [hooks.common.modules :as modules])+ (:import+ (java.io BufferedInputStream ByteArrayOutputStream File OutputStream)+ (java.nio.charset StandardCharsets)+ (java.nio.file CopyOption Files StandardCopyOption)+ (java.security MessageDigest)+ (java.time LocalDate ZoneOffset)+ (java.util BitSet)))++(set! *warn-on-reflection* true)++;;;; Git++(defn- git!+ "Run git with `args` in the repo and return its output, throwing when it fails."+ [args]+ (let [{:keys [exit out err]} (apply sh/sh "git" args)]+ (when-not (zero? exit)+ (throw (ex-info (format "git %s failed: %s" (str/join " " args) err) {:args args})))+ out))++(defn- open-object-reader+ "A running `git cat-file --batch` for reading object contents with [[read-object]].+ Close it with [[close-object-reader]]."+ []+ (let [process (.start (ProcessBuilder. ^"[Ljava.lang.String;" (into-array String ["git" "cat-file" "--batch"])))]+ {:process process+ :in (BufferedInputStream. (.getInputStream process))+ :out (.getOutputStream process)}))++(defn- close-object-reader [{:keys [^Process process ^OutputStream out]}]+ (.close out)+ (.waitFor process))++(defn- read-line-bytes+ "One `\\n`-terminated line from `in`, without the newline."+ [^BufferedInputStream in]+ (let [buffer (ByteArrayOutputStream.)]+ (loop []+ (let [b (.read in)]+ (if (or (= b -1) (= b (int \newline)))+ (String. (.toByteArray buffer) StandardCharsets/UTF_8)+ (do (.write buffer b) (recur)))))))++(defn- read-object+ "Contents of the object `spec` names, a blob sha or `commit:path`, as a string; `nil` when it does not exist."+ [{:keys [^BufferedInputStream in ^OutputStream out]} spec]+ (.write out (.getBytes (str spec "\n") StandardCharsets/UTF_8))+ (.flush out)+ (let [header (read-line-bytes in)]+ (when-not (str/ends-with? header " missing")+ (let [size (parse-long (last (str/split header #" ")))+ bytes (.readNBytes in (int size))]+ ;; Each object is followed by a newline.+ (.read in)+ (String. bytes StandardCharsets/UTF_8)))))++(def ^:private source-path+ "Paths [[dev.deps-graph/dependencies]] scans: OSS and EE sources, every driver, and the plugin modules."+ #"^(?:src|enterprise/backend/src|modules/drivers/[^/]+/src|modules/embedder/src)/.+\.cljc?$")++(defn- source-blobs+ "The `{:path, :blob}` of each backend source file in `commit`."+ [commit]+ (into []+ (keep (fn [entry]+ ;; Each entry is `<mode> <type> <sha>\t<path>`.+ (let [[meta-part path] (str/split entry #"\t" 2)+ [_mode kind blob] (str/split meta-part #" ")]+ (when (and (= kind "blob") path (re-matches source-path path))+ {:path path, :blob blob}))))+ (str/split (git! ["ls-tree" "-r" "-z" commit+ "--" "src" "enterprise/backend/src" "modules/drivers" "modules/embedder"])+ #"\u0000")))++;;;; Module config++(defn- legacy-module-name+ "Today's name for the module the old linter keyed by its root namespace: `metabase.lib` is `lib`, and+ `metabase-enterprise.sso` is `enterprise/sso`.+ Returns nil for a namespace outside both roots."+ [root-ns]+ (when-let [[_ enterprise module] (re-matches #"metabase(-enterprise)?\.(.+)" (str root-ns))]+ (if enterprise (symbol "enterprise" module) (symbol module))))++(defn- legacy-modules+ "The module map from the `:metabase/ns-module-checker` linter that predates `:metabase/modules`.+ That linter keyed `:api-namespaces` and `:allowed-modules` by a module's root namespace, such as `metabase.lib`.+ It had no friends or nesting, and its `:allowed-modules` becomes `:uses`."+ [config-source]+ ;; Before #53583 an unlisted module's API was its root namespace; today's default is `.api`, `.core` and `.init`,+ ;; so exposure measured against this config is approximate.+ (let [linter (get-in (binding [*read-eval* false] (read-string config-source))+ [:linters :metabase/ns-module-checker])]+ (into {}+ (for [[root-ns api] (:api-namespaces linter)+ :let [module (legacy-module-name root-ns)+ uses (get-in linter [:allowed-modules root-ns])]+ ;; Same exclusion as [[deps-graph/kondo-config]].+ :when (and module (not= 'connection-pool module))]+ [module (cond-> {}+ (some? api) (assoc :api (if (= :any api) :any (set api)))+ (some? uses) (assoc :uses (if (= :any uses) :any (into #{} (keep legacy-module-name) uses))))]))))++(def ^:private whole-backend+ "The module map before #42250, when there were no modules: the whole backend is one."+ '{backend {}})++(def ^:private modules-config-path ".clj-kondo/config/modules/config.edn")++(def ^:private legacy-config-path+ "Where the module linter's config lived before #53583 gave it [[modules-config-path]]."+ ".clj-kondo/config.edn")++(defn- commit-modules+ "The `:metabase/modules` map of a commit, where `read-file` returns the contents of a path in it, or nil.+ Before #53583 it is translated from the older linter config.+ Before #42250 there is no module config, so it is [[whole-backend]]."+ [read-file]+ (if-let [config (read-file modules-config-path)]+ ;; Same exclusion as [[deps-graph/kondo-config]].+ (dissoc (:metabase/modules (edn/read-string config)) 'connection-pool)+ (or (not-empty (some-> (read-file legacy-config-path) legacy-modules))+ whole-backend)))++(defn- prefix->module+ "Namespace prefixes for `modules`. [[whole-backend]] owns every OSS and EE namespace, and `metabase-init`, the+ only namespace of the repository's first commit."+ [modules]+ (if (= modules whole-backend)+ {"metabase" 'backend, "metabase-enterprise" 'backend, "metabase-init" 'backend}+ (modules/build-prefix->module modules)))++;;;; Cache++(defn cache-dir+ "Root of the on-disk cache: `$MB_MODULE_HISTORY_CACHE`, else `~/.cache/metabase/module-history`."+ ^File []+ (io/file (or (System/getenv "MB_MODULE_HISTORY_CACHE")+ (str (System/getProperty "user.home") "/.cache/metabase/module-history"))))++(defn- code-hash+ "A short hash over the source of classpath `resources`, naming a cache layer computed by that code."+ [resources]+ (let [digest (MessageDigest/getInstance "SHA-1")]+ (doseq [resource resources]+ (.update digest (.getBytes (slurp (or (io/resource resource)+ (throw (ex-info (str "Not on the classpath: " resource) {}))))+ StandardCharsets/UTF_8)))+ (subs (format "%040x" (BigInteger. 1 (.digest digest))) 0 12)))++(def ^:private parse-version+ (delay (code-hash ["dev/module_source.clj"])))++(def ^:private metrics-version+ (delay (code-hash ["dev/deps_graph.clj"+ "dev/module_graph.clj"+ "dev/module_history.clj"+ "dev/module_source.clj"+ "hooks/common/modules.clj"])))++(defn- read-cached [^File file]+ (when (.exists file)+ (edn/read-string (slurp file))))++(defn- write-cached!+ "Write `value` to `file` through a temporary file, so concurrent runs never read half a file."+ [^File file value]+ (io/make-parents file)+ (let [tmp (File/createTempFile "entry" ".tmp" (.getParentFile file))]+ (spit tmp (pr-str value))+ (Files/move (.toPath tmp) (.toPath file) (into-array CopyOption [StandardCopyOption/ATOMIC_MOVE]))+ value))++(defn- parse-file ^File [blob]+ (io/file (cache-dir) "parse" @parse-version (subs blob 0 2) (str blob ".edn")))++(defn- metrics-file ^File [commit]+ (io/file (cache-dir) "metrics" @metrics-version (str commit ".edn")))++(defn- parse-source+ "The [[module-source/source-requires]] map for `source`, or `{:error message}` when it does not parse."+ [source]+ (try+ (module-source/source-requires source)+ (catch Exception e+ {:error (ex-message e)})))++(defn- blob-requires+ "Requires for each of `blobs`, from the cache where possible.+ Missing blobs are read in turn and parsed in parallel."+ [reader blobs]+ (let [cached (into {} (keep (fn [blob] (some->> (read-cached (parse-file blob)) (vector blob)))) blobs)+ missing (remove cached blobs)+ sources (mapv (fn [blob] [blob (read-object reader blob)]) missing)+ parsed (into {} (pmap (fn [[blob source]]+ [blob (write-cached! (parse-file blob) (parse-source source))])+ sources))]+ (merge cached parsed)))++;;;; Metrics++(defn commit-dependencies+ "Dependency rows, as [[deps-graph/dependencies]] returns them, for the sources at `commit` under `modules`."+ [reader commit modules]+ (let [prefix->module (prefix->module modules)+ files (source-blobs commit)+ requires (blob-requires reader (distinct (map :blob files)))]+ (into []+ (keep (fn [{:keys [path blob]}]+ (let [{ns-symb :namespace, :as parsed} (requires blob)]+ ;; Unparseable files and namespaces outside every module are skipped, as a checkout scan cannot+ ;; place them either.+ (when (and ns-symb+ (not (:error parsed))+ (modules/resolve-module prefix->module ns-symb))+ (deps-graph/resolve-requires prefix->module path parsed)))))+ files)))++(defn- largest-cycle+ "The `[modules namespaces]` sizes of the cycle with the most namespaces, or `[0 0]` without cycles."+ [module-sizes namespace-sizes]+ (or (last (sort-by second (map vector module-sizes namespace-sizes)))+ [0 0]))++(defn- on-big-stack+ "Call `f` on a thread with a 512 MB stack and return its result."+ [f]+ ;; The namespace graph has about 2,000 nodes, and the recursive cycle search goes one level deeper per node on+ ;; its path, which can outgrow the default 2 MB stack.+ (let [result (promise)+ thread (Thread. nil+ #(deliver result (try {:ok (f)} (catch Throwable e {:error e})))+ "module-history-scc"+ (* 512 1024 1024))]+ (.start thread)+ (let [{:keys [ok error]} @result]+ (when error+ (throw error))+ ok)))++(defn- structure-metrics+ "The measures stats dashboard 9805 tracks, defined as when it was built: cycle sizes in+ [[module-graph/cyclic-components]] order, the largest namespace-level cycle, module pairs that require each other,+ and cross-module requires past a module's `:api`, including those a `:friends` grant allows.+ The `graph` and `ns-graph` map each module and each namespace to the ones it requires."+ [deps modules stats graph ns-graph]+ (let [api-of (memoize (fn [module]+ ;; The declared API: nil for `:api :any`, else the declared set, else the default+ ;; namespaces that exist.+ (let [api (get-in modules [module :api])]+ (cond+ (= :any api) nil+ (set? api) api+ :else (let [prefix (modules/module-ns-prefix modules module)]+ (into #{}+ (filter #(contains? ns-graph %))+ [(symbol (str prefix ".api"))+ (symbol (str prefix ".core"))+ (symbol (str prefix ".init"))]))))))+ non-api (for [{:keys [module namespace], :as row} deps+ {to-ns :namespace, to :module} (:deps row)+ :when (not= module to)+ :let [api (api-of to)]+ :when (and (some? api) (not (contains? api to-ns)))]+ {:from-ns namespace, :from module, :to-ns to-ns, :to to})+ via-friends (filter (fn [{:keys [from to]}] (contains? (set (get-in modules [to :friends])) from)) non-api)+ mutual (for [[a bs] graph+ b bs+ :when (and (neg? (compare (str a) (str b)))+ (contains? (get graph b) a)+ (not (modules/descendant-of? modules a b))+ (not (modules/descendant-of? modules b a)))]+ [a b])+ size-at (fn [sizes i] (get sizes i 0))]+ {:distinct-namespaces (count ns-graph)+ :configured-top-level-modules (count (remove #(modules/parent-module modules %) (keys modules)))+ :scc1-modules (size-at (:scc-module-sizes stats) 0)+ :scc1-namespaces (size-at (:scc-namespace-sizes stats) 0)+ :scc2-modules (size-at (:scc-module-sizes stats) 1)+ :scc2-namespaces (size-at (:scc-namespace-sizes stats) 1)+ :scc3-modules (size-at (:scc-module-sizes stats) 2)+ :scc3-namespaces (size-at (:scc-namespace-sizes stats) 2)+ :largest-ns-level-scc (count (first (on-big-stack #(module-graph/cyclic-components ns-graph))))+ :mutual-dependency-pairs (count mutual)+ :friend-internal-namespaces (count (into #{} (map :to-ns) via-friends))+ :friend-internal-requires (count (into #{} (map (juxt :from-ns :to-ns)) via-friends))+ :non-api-cross-module-requires (count (into #{} (map (juxt :from-ns :to-ns)) non-api))+ :total-api-namespaces (transduce (map #(count (or (api-of %) #{}))) + 0 (keys modules))+ :ns-prefixes (count (keep :ns-prefix (vals modules)))+ :model-imports-bypass (count (filter #(= :bypass (:model-imports %)) (vals modules)))+ :legacy-rest-modules (count (filter #(re-find #"-rest$" (str %)) (keys modules)))}))++;;;; Retest share++(defn- dependents+ "The nodes a change to each node could break: every node that reaches it through requires, itself included.+ The `graph` maps a node to the nodes it requires.+ Returns `{:nodes [node ...], :sets [BitSet ...]}`, one set per node, whose bits index into `:nodes`."+ [graph]+ (let [nodes (vec (into (set (keys graph)) cat (vals graph)))+ index (zipmap nodes (range))+ sets (mapv (fn [i] (doto (BitSet. (count nodes)) (.set (int i)))) (range (count nodes)))+ edges (vec (for [[user requires] graph+ required requires]+ [(sets (index user)) (sets (index required))]))]+ ;; Whatever reaches a user also reaches what it requires. Repeat until a pass adds nothing, which takes about+ ;; as many passes as the longest chain of requires.+ (loop []+ (when (reduce (fn [changed? [^BitSet user ^BitSet required]]+ (let [before (.cardinality required)]+ (.or required user)+ (or changed? (not= before (.cardinality required)))))+ false+ edges)+ (recur)))+ {:nodes nodes, :sets sets}))++(defn- set-bits+ "The indices of the set bits of `bits`, in ascending order."+ [^BitSet bits]+ (take-while nat-int? (iterate #(.nextSetBit bits (inc (long %))) (.nextSetBit bits 0))))++(defn- percent+ "The share `part` is of `whole`, as a percentage with one decimal."+ [part whole]+ (if (zero? whole) 0.0 (/ (Math/round (* 1000.0 (/ part whole))) 10.0)))++(defn- share-summary+ "The mean and median of `shares`, percentages with one entry per namespace, and how many fall in each tenth.+ The median of an even number of shares is the upper of the two middle values."+ [shares]+ (let [sorted (vec (sort shares))+ total (count sorted)]+ {:mean (if (zero? total) 0.0 (/ (Math/round (* 10.0 (/ (reduce + sorted) total))) 10.0))+ :median (if (zero? total) 0.0 (sorted (quot total 2)))+ :histogram (let [tenths (frequencies (map #(min 9 (int (/ % 10))) sorted))]+ (mapv #(get tenths % 0) (range 10)))}))++(defn- retest-metrics+ "How much of the backend a change would retest, taking every namespace in turn as the one that changed.+ A test run selected by module reruns every namespace of every module that reaches the changed namespace's module;+ one selected by namespace reruns only the namespaces that reach it.+ Shares are percentages of all namespaces.++ A leaf is a module or namespace nothing else requires, so a change to it retests only itself.++ The `graph` and `ns-graph` map each module and each namespace to the ones it requires.+ The `total` is the number of rows in `deps`."+ [deps graph ns-graph total]+ (let [mod->count (frequencies (map :module deps))+ by-namespace (dependents ns-graph)+ {:keys [nodes sets]} (dependents graph)+ module-share (zipmap nodes+ ;; A module that is required but has no source in `deps` has no namespaces to count.+ (map #(percent (transduce (map (fn [i] (mod->count (nodes i) 0))) + 0 (set-bits %)) total)+ sets))+ ;; Something requires every module without a source in `deps`, so each leaf has namespaces.+ leaf-modules (keep (fn [[module ^BitSet reaching]] (when (= 1 (.cardinality reaching)) module))+ (map vector nodes sets))+ per-module (share-summary (map (comp module-share :module) deps))+ per-namespace (share-summary (map (fn [^BitSet reaching] (percent (.cardinality reaching) total))+ (:sets by-namespace)))]+ {:retest-module-mean (:mean per-module)+ :retest-module-median (:median per-module)+ :retest-namespace-mean (:mean per-namespace)+ :retest-namespace-median (:median per-namespace)+ :retest-histogram {:module (:histogram per-module), :namespace (:histogram per-namespace)}+ :leaf-modules (count leaf-modules)+ :leaf-module-namespaces (transduce (map mod->count) + 0 leaf-modules)+ :leaf-namespaces (count (filter (fn [^BitSet reaching] (= 1 (.cardinality reaching)))+ (:sets by-namespace)))}))++(defn commit-metrics+ "Metrics for one commit's dependency rows and module map: [[deps-graph/module-boundary-stats]] plus the sizes,+ cycle, exposure and `:friends` measures the history report tracks, and the [[structure-metrics]].+ All are numbers except `:cycles`, one map per module cycle with its `:modules`, `:namespaces`, `:edges`,+ `:density`, `:cycle-rank` and `:strong-bridges` count, largest first, and `:retest-histogram`, the namespaces in+ each tenth of retest share; see [[retest-metrics]]."+ [deps modules]+ (let [graph (deps-graph/module-dependencies deps)+ cycles (module-graph/cycles graph)+ stats (deps-graph/module-boundary-stats deps modules cycles)+ existing (into #{} (map :namespace) deps)+ ns-graph (into {}+ (map (fn [{:keys [namespace deps]}]+ [namespace (into #{} (comp (map :namespace) (filter existing)) deps)]))+ deps)+ mod->nses (update-vals (group-by :module deps) #(into #{} (map :namespace) %))+ ns-count #(count (get mod->nses % #{}))+ with-nses (set (keys mod->nses))+ root-of (memoize (fn [m] (last (take-while some? (iterate #(modules/parent-module modules %) m)))))+ api-nses (fn [m]+ (let [nses (get mod->nses m #{})]+ (if (= :any (get-in modules [m :api]))+ nses+ (into #{} (filter (set (modules/module-api-namespaces modules m))) nses))))+ ;; A module outside every subtree may name exactly the modules visible across top-level modules.+ exported? #(nil? (modules/namability-error modules 'module-history/outsider %))+ non-api #(- (ns-count %) (count (api-nses %)))+ cross-friends (fn [m] (remove #(= (root-of %) (root-of m)) (get-in modules [m :friends])))+ [largest-m largest-n] (largest-cycle (:scc-module-sizes stats) (:scc-namespace-sizes stats))+ static-graph (deps-graph/module-dependencies (map (fn [row] (update row :deps #(remove :dynamic %))) deps))+ static-n (transduce (map (fn [component] (reduce + (map ns-count component)))) max 0+ (module-graph/cyclic-components static-graph))+ top-edges (into #{}+ (for [[from tos] graph+ to tos+ :when (not= (root-of from) (root-of to))]+ [(root-of from) (root-of to)]))+ ns->mod (into {} (map (juxt :namespace :module)) deps)+ namespaces (count deps)]+ (merge+ (structure-metrics deps modules stats graph ns-graph)+ (retest-metrics deps graph ns-graph namespaces)+ ;; Each cycle's members, so a report can follow a cycle as it splits or merges.+ {:cycles (mapv (fn [{members :modules, :keys [edges cycle-rank strong-bridges]} ns-size density]+ {:modules members+ :namespaces ns-size+ :edges edges+ :density density+ :cycle-rank cycle-rank+ :strong-bridges (count strong-bridges)})+ cycles+ (:scc-namespace-sizes stats)+ (:scc-densities stats))}+ (select-keys stats [:api-any-namespaces+ :upward-requires+ :upward-module-edges+ :upward-internal-requires+ :upward-cycle-requires])+ {:namespaces namespaces+ :modules (count with-nses)+ :configured-modules (count modules)+ :top-level-modules (count (into #{} (map root-of) with-nses))+ :nested-modules (count (remove #(= % (root-of %)) with-nses))+ :module-edges (reduce + (map count (vals graph)))+ :top-level-edges (count top-edges)+ :scc-largest-modules largest-m+ :scc-largest-namespaces largest-n+ :scc-largest-pct (percent largest-n namespaces)+ :scc-cyclic-modules (reduce + (:scc-module-sizes stats))+ :scc-cyclic-namespaces (reduce + (:scc-namespace-sizes stats))+ :scc-components (count (:scc-module-sizes stats))+ :scc-static-largest-namespaces static-n+ :scc-dynamic-only-namespaces (- largest-n static-n)+ :api-any-modules (count (filter #(= :any (get-in modules [% :api])) with-nses))+ :uses-any-modules (count (filter #(= :any (:uses %)) (vals modules)))+ ;; Namespaces another top-level module may require: the API of every module visible outside its subtree.+ :exposed-namespaces (reduce + (map #(count (api-nses %)) (filter exported? with-nses)))+ :hidden-modules (count (remove exported? with-nses))+ :hidden-namespaces (reduce + (map ns-count (remove exported? with-nses)))+ :cross-top-level-required-namespaces (count (into #{}+ (for [{:keys [module], :as row} deps+ {required :namespace} (:deps row)+ :let [to (ns->mod required)]+ :when (and to (not= (root-of module) (root-of to)))]+ required)))+ :friend-grants (reduce + (map #(count (:friends %)) (vals modules)))+ :cross-top-level-friend-grants (reduce + (map #(count (cross-friends %)) (keys modules)))+ ;; Non-API namespaces opened to another top-level module through `:friends`.+ :friend-exposed-namespaces (reduce + (map non-api (filter #(seq (cross-friends %)) with-nses)))+ :friend-access-paths (reduce + (map #(* (count (cross-friends %)) (non-api %)) with-nses))})))++;;;; Commits++(defn- commit-info+ "The `:sha`, `:committed-at` and `:subject` of `commit`."+ [commit]+ (let [line (str/trim-newline (git! ["show" "-s" "--format=%H%x00%cI%x00%s" commit]))+ [sha committed-at subject] (str/split line #"\u0000" 3)]+ {:sha sha, :committed-at committed-at, :subject subject}))++(defn- resolve-commit+ "The full sha of `commit`, throwing when it names no commit."+ [commit]+ (str/trim (git! ["rev-parse" (str commit "^{commit}")])))++(defn- commit-snapshot+ "The dependency rows and module map at the commit `sha`, as `{:deps rows, :modules map}`."+ [reader sha]+ (let [modules (commit-modules #(read-object reader (str sha ":" %)))]+ {:deps (commit-dependencies reader sha modules)+ :modules modules}))++(defn measure+ "Metrics for `commit`, from the cache when this code has measured it before.+ The `:module-system?` key is false before #42250, when the whole backend counts as one module."+ ([commit]+ (let [reader (open-object-reader)]+ (try+ (measure reader commit)+ (finally+ (close-object-reader reader)))))+ ([reader commit]+ (let [sha (resolve-commit commit)+ file (metrics-file sha)]+ (or (read-cached file)+ (let [{:keys [deps modules]} (commit-snapshot reader sha)]+ (write-cached! file (merge (commit-info sha)+ {:module-system? (not= modules whole-backend)}+ (commit-metrics deps modules))))))))++(defn snapshot+ "The dependency rows and module map at `commit`, as `{:deps rows, :modules map}`.+ For analyses of one commit that [[measure]] does not record."+ [commit]+ (let [reader (open-object-reader)]+ (try+ (commit-snapshot reader (resolve-commit commit))+ (finally+ (close-object-reader reader)))))++(def ^:private source-pathspecs+ "Pathspecs for the backend sources [[source-path]] matches."+ ;; A `*` in a plain pathspec does not stop at `/`, so `modules/drivers/*/src` matches no file under a driver's+ ;; `src` directory. The glob form does.+ ["src" "enterprise/backend/src" ":(glob)modules/drivers/*/src/**" "modules/embedder/src"])++(defn- midnight-utc+ "The instant `date` starts in UTC, in the form git reads."+ [date]+ ;; Git reads a bare date as that day at the current local time, so a sample would differ from run to run.+ (str date "T00:00:00Z"))++(defn- commits-since+ "First-parent commits of `ref` since `date` that touch any of `pathspecs`, newest first."+ [ref date pathspecs]+ (let [args ["log" "--first-parent" "--format=%H" (str "--since=" (midnight-utc date)) ref "--"]]+ (remove str/blank? (str/split-lines (git! (into args pathspecs))))))++(defn- config-commits+ "Commits of `ref` since `date` that changed the module config.+ Before [[modules-config-path]] existed that is a change to [[legacy-config-path]], which is followed only+ up to the commit that moved the config out of it, or to `ref` when `ref` predates that commit."+ [ref date]+ (let [introduced (last (str/split-lines (git! ["log" "--first-parent" "--diff-filter=A" "--format=%H" ref+ "--" modules-config-path])))]+ (concat (commits-since ref date [modules-config-path])+ (commits-since (if (str/blank? introduced) ref (str introduced "^")) date [legacy-config-path]))))++(defn- last-commit-before+ "The last first-parent commit of `ref` before each of `dates`."+ [ref dates]+ (keep #(not-empty (str/trim (git! ["rev-list" "-1" "--first-parent" (str "--before=" (midnight-utc %)) ref])))+ dates))++(defn- dates-from+ "Every date from `start` up to today in UTC, `step` apart."+ [^LocalDate start step]+ (let [today (LocalDate/now ZoneOffset/UTC)]+ (take-while #(not (.isAfter ^LocalDate % today)) (iterate step start))))++(defn sample-commits+ "Commits to measure on the first-parent history of `ref` (default `origin/master`), oldest first:++ - every one since `since` that changed the module config, which changes on every boundary shift+ - every one since `backend-since` that changed a backend source or the module config+ - the last commit before each week from `weekly-since`, and before the first of each month from `monthly-since`+ - each of `extra`, and the tip of `ref`++ Dates are days in UTC."+ [{:keys [ref since backend-since weekly-since monthly-since extra], :or {ref "origin/master"}}]+ (let [changed (concat (when since (config-commits ref since))+ (when backend-since (concat (commits-since ref backend-since source-pathspecs)+ (config-commits ref backend-since))))+ weeks (when weekly-since+ (dates-from (LocalDate/parse weekly-since) #(.plusWeeks ^LocalDate % 1)))+ months (when monthly-since+ (dates-from (.withDayOfMonth (LocalDate/parse monthly-since) 1) #(.plusMonths ^LocalDate % 1)))+ anchors (last-commit-before ref (concat weeks months))+ resolved (map resolve-commit (concat changed anchors extra [ref]))+ order (into {}+ (map-indexed (fn [i sha] [sha i]))+ (str/split-lines (git! ["rev-list" "--first-parent" "--reverse" ref])))]+ (->> (distinct resolved)+ (sort-by #(order % Long/MAX_VALUE))+ vec)))++;;;; Backfill++(def ^:private columns+ "Output columns, in order."+ [:sha :committed-at :subject :module-system?+ :namespaces :modules :configured-modules :top-level-modules :nested-modules :module-edges :top-level-edges+ :scc-largest-modules :scc-largest-namespaces :scc-largest-pct :scc-cyclic-modules :scc-cyclic-namespaces+ :scc-components :scc-static-largest-namespaces :scc-dynamic-only-namespaces+ :upward-requires :upward-module-edges :upward-internal-requires :upward-cycle-requires+ :exposed-namespaces :hidden-modules :hidden-namespaces :cross-top-level-required-namespaces+ :api-any-modules :api-any-namespaces :uses-any-modules+ :friend-grants :cross-top-level-friend-grants :friend-exposed-namespaces :friend-access-paths+ :distinct-namespaces :configured-top-level-modules+ :scc1-modules :scc1-namespaces :scc2-modules :scc2-namespaces :scc3-modules :scc3-namespaces+ :largest-ns-level-scc :mutual-dependency-pairs :friend-internal-namespaces :friend-internal-requires+ :non-api-cross-module-requires :total-api-namespaces :ns-prefixes :model-imports-bypass :legacy-rest-modules+ :retest-module-mean :retest-module-median :retest-namespace-mean :retest-namespace-median+ :leaf-modules :leaf-module-namespaces :leaf-namespaces])++(defn- commits-to-measure+ "The shas [[backfill!]] measures for `opts`: each of `commits` and, when a `-since` option is given or `commits`+ is empty, the [[sample-commits]], without repeats."+ [{:keys [commits], :as opts}]+ (if (seq commits)+ ;; Resolve first, so `HEAD` and the sampled tip it names are measured once.+ (vec (distinct (concat (map resolve-commit commits)+ (when (some opts [:since :backend-since :weekly-since :monthly-since])+ (sample-commits opts)))))+ (sample-commits opts)))++(defn backfill!+ "Measure the commits [[sample-commits]] picks for `opts` and return the rows.+ With `:commits`, measure those, adding the sample only when a `-since` option is given.+ With `:out`, also write them there as CSV, or as one EDN map per line when the name ends in `.edn`.++ clojure -X:dev dev.module-history/backfill! \\+ :since '\"2025-10-01\"' :monthly-since '\"2023-12-01\"' :out '\"history.csv\"'"+ [{:keys [out], :as opts}]+ (let [shas (commits-to-measure opts)+ reader (open-object-reader)+ rows (try+ (into []+ (map-indexed (fn [i sha]+ (let [row (measure reader sha)]+ ;; An explicit commit can be any revision, such as `HEAD`: print what it+ ;; resolved to.+ (println (format "%d/%d %s %s largest cycle %d" (inc i) (count shas)+ (subs (:sha row) 0 11) (:committed-at row)+ (:scc-largest-namespaces row)))+ row)))+ shas)+ (finally+ (close-object-reader reader)))]+ (when out+ (if (str/ends-with? (str out) ".edn")+ (spit out (str/join (map #(str (pr-str %) "\n") rows)))+ (with-open [writer (io/writer out)]+ (csv/write-csv writer (cons (map name columns)+ (map (fn [row] (map #(get row % "") columns)) rows)))))+ (println "Wrote" (count rows) "rows to" out))+ rows))++(comment+ (measure "origin/master")+ (sample-commits {:since "2026-09-01"})+ (backfill! {:since "2025-10-01", :monthly-since "2023-12-01", :out "/tmp/module-history.csv"}))diff --git a/dev/src/dev/module_source.clj b/dev/src/dev/module_source.cljnew file mode 100644index 00000000000..2d64047ec77--- /dev/null+++ b/dev/src/dev/module_source.clj@@ -0,0 +1,81 @@+(ns dev.module-source+ "Which namespaces a Clojure source file requires: statically in its `ns` form, and dynamically through `require`,+ `requiring-resolve` and their aliases.++ [[dev.module-history]] caches parsed files under a hash of this file, so code that does not affect parsing belongs+ elsewhere."+ (:require+ [clojure.tools.namespace.parse :as ns.parse]+ [rewrite-clj.node :as n]+ [rewrite-clj.parser :as r.parser])+ (:import+ (java.io PushbackReader StringReader)))++(set! *warn-on-reflection* true)++(defn skip-node?+ "Whether a rewrite-clj `node` is whitespace, a comment, a `#_` form or a `(comment ...)` block."+ [node]+ (let [tag (n/tag node)]+ (or (#{:uneval :comment :newline :whitespace} tag)+ (and (= tag :list)+ (when-let [fst (first (n/children node))]+ (and (= (n/tag fst) :token)+ (= (:value fst) 'comment)))))))++(defn walk-parsed-ignore-comments!+ "Call `f`, for its side effects, on each node of `tree`, a rewrite-clj parse, skipping comments and their contents."+ [f tree]+ (letfn [(walk [node]+ (when-not (skip-node? node)+ ;; Don't descend into comments+ (f node)+ (when-let [children (:children node)]+ (run! walk children))))]+ (walk tree)))++(def ^:private require-symbols+ '#{require+ clojure.core/require+ classloader/require+ metabase.classloader.core/require+ metabase.classloader.impl/require+ requiring-resolve+ clojure.core/requiring-resolve})++(defn dynamic-requires+ "Namespaces that `(require ...)` forms and their aliases load anywhere in `tree`, a rewrite-clj parse.+ Comments are ignored."+ [tree]+ (let [acc (atom #{})]+ (walk-parsed-ignore-comments!+ (fn [node]+ (when (= (n/tag node) :list)+ (when-let [[first-child & other :as children] (remove skip-node? (n/children node))]+ (when (and first-child+ (= (n/tag first-child) :token)+ ;; Check all child symbols on the line since `require` might be called in a threading macro+ ;; like (-> 'ns require)+ (some #(and (= (n/tag %) :token)+ (require-symbols (n/sexpr %)))+ children))+ (run! #(when (= (n/tag %) :quote)+ (when-let [in (some-> (n/children %) first n/sexpr)]+ (when (symbol? in)+ (swap! acc conj (if (qualified-symbol? in)+ (symbol (namespace in))+ in)))))+ other)))))+ tree)+ @acc))++(defn source-requires+ "The namespace that Clojure `source` text declares and the namespaces it requires, as+ `{:namespace ns, :static #{...}, :dynamic #{...}}`.+ The `:namespace` is `nil` when the source has no `ns` form."+ [^String source]+ (let [decl (with-open [reader (PushbackReader. (StringReader. source))]+ (ns.parse/read-ns-decl reader))]+ {:namespace (some-> decl ns.parse/name-from-ns-decl)+ :static (set (some-> decl ns.parse/deps-from-ns-decl))+ :dynamic (dynamic-requires (r.parser/parse-string-all source))}))diff --git a/dev/test/metabase/core/module_history_test.clj b/dev/test/metabase/core/module_history_test.cljnew file mode 100644index 00000000000..47fa95bc064--- /dev/null+++ b/dev/test/metabase/core/module_history_test.clj@@ -0,0 +1,298 @@+(ns metabase.core.module-history-test+ (:require+ [clojure.set :as set]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [dev.deps-graph :as deps-graph]+ [dev.module-history :as module-history]+ [dev.module-source :as module-source]+ [hooks.common.modules]))++(set! *warn-on-reflection* true)++(deftest ^:parallel source-requires-test+ (testing "static requires come from the ns form, dynamic ones from require calls outside comments"+ (is (= '{:namespace metabase.x.core+ :static #{metabase.a.core metabase.b.core}+ :dynamic #{metabase.d.core metabase.f.core}}+ (module-source/source-requires+ (str "(ns metabase.x.core (:require [metabase.a.core :as a] [metabase.b.core]))\n"+ "(comment (require 'metabase.c.core))\n"+ "(defn g [] (requiring-resolve 'metabase.d.core/h))\n"+ "(defn i [] (-> 'metabase.f.core require))\n")))))+ (testing "a source without an ns form has no namespace"+ (is (= {:namespace nil, :static #{}, :dynamic #{}}+ (module-source/source-requires "(def x 1)")))))++(deftest ^:parallel resolve-requires-test+ (let [prefix->module (#'module-history/prefix->module '{a {}, b {}, config {}, test {}})]+ (testing "a namespace required both ways counts as static, and one outside every module is left out"+ (is (= '{:namespace metabase.a.core+ :filename "src/metabase/a/core.clj"+ :module a+ :deps [{:namespace metabase.b.core, :module b}+ {:namespace metabase.b.impl, :module b, :dynamic :require-and-friends}]}+ (deps-graph/resolve-requires prefix->module+ "src/metabase/a/core.clj"+ '{:namespace metabase.a.core+ :static #{metabase.b.core clojure.string}+ :dynamic #{metabase.b.core metabase.b.impl}}))))+ (testing "the config namespace's probe for test code is not a dependency, under its old name too"+ (is (= [[] []]+ (for [config-namespace '[metabase.config metabase.config.core]]+ (:deps (deps-graph/resolve-requires prefix->module+ "src/metabase/config.clj"+ {:namespace config-namespace+ :static #{}+ :dynamic '#{metabase.test.dummy-namespace}}))))))))++(deftest ^:parallel whole-backend-test+ (testing "before the module system every OSS and EE namespace belongs to one module"+ (let [prefix->module (#'module-history/prefix->module @#'module-history/whole-backend)]+ (is (= '[backend backend backend backend]+ (mapv #(hooks.common.modules/resolve-module prefix->module %)+ ;; `metabase-init.core` is the only namespace of the repository's first commit.+ '[metabase.lib.core metabase.query-processor.middleware.x metabase-enterprise.sso.core+ metabase-init.core]))))))++(deftest ^:parallel legacy-modules-test+ (testing "the pre-:metabase/modules linter config translates root namespaces to module names"+ (is (= '{api {:api :any, :uses :any}+ enterprise/sso {:api #{metabase-enterprise.sso.core}, :uses #{api lib}}+ lib {:api #{metabase.lib.core}, :uses #{}}+ setup {}}+ (#'module-history/legacy-modules+ (pr-str '{:linters {:metabase/ns-module-checker+ {:api-namespaces {metabase.api :any+ metabase.lib #{metabase.lib.core}+ metabase-enterprise.sso #{metabase-enterprise.sso.core}+ metabase.setup nil+ ;; The modern config leaves this module out, so this does too.+ metabase.connection-pool #{metabase.connection-pool}}+ :allowed-modules {metabase.api :any+ metabase.lib #{}+ metabase-enterprise.sso #{metabase.api metabase.lib}+ ;; Not a module, since it has no :api-namespaces entry.+ metabase.unlisted :any}}}}))))))++(deftest source-pathspecs-test+ (testing "each pathspec used to find backend-touching commits matches files in this tree"+ (is (= []+ (remove (fn [pathspec] (seq (#'module-history/git! ["ls-files" "--" pathspec])))+ @#'module-history/source-pathspecs))))+ (testing "a driver's sources are covered"+ (is (some #(re-find #"^modules/drivers/[^/]+/src/.+\.clj$" %)+ (str/split-lines (#'module-history/git! (into ["ls-files" "--"] @#'module-history/source-pathspecs)))))))++(deftest ^:parallel midnight-utc-test+ (testing "a sampling date is pinned to the start of its day, so git does not read it at the current time of day"+ (is (= "2023-12-01T00:00:00Z"+ (#'module-history/midnight-utc "2023-12-01")))))++(deftest ^:parallel commit-metrics-test+ ;; Module x.y is nested in x, which does not export it; z lets x into its internals; w has `:api :any`.+ ;; Module z requires x only through `requiring-resolve`, so the static-only cycle is smaller than the full one.+ (let [deps '[{:namespace metabase.x.core+ :filename "src/metabase/x/core.clj"+ :module x+ :deps [{:namespace metabase.x.y.core, :module x.y}+ {:namespace metabase.z.impl, :module z}]}+ {:namespace metabase.x.impl+ :filename "src/metabase/x/impl.clj"+ :module x+ :deps []}+ {:namespace metabase.x.y.core+ :filename "src/metabase/x/y/core.clj"+ :module x.y+ :deps [{:namespace metabase.x.core, :module x}]}+ {:namespace metabase.z.core+ :filename "src/metabase/z/core.clj"+ :module z+ :deps [{:namespace metabase.x.core, :module x, :dynamic :require-and-friends}]}+ {:namespace metabase.z.impl+ :filename "src/metabase/z/impl.clj"+ :module z+ :deps []}+ {:namespace metabase.w.core+ :filename "src/metabase/w/core.clj"+ :module w+ :deps [{:namespace metabase.z.core, :module z}]}]+ modules '{w {:api :any}+ x {:api #{metabase.x.core}}+ x.y {}+ z {:friends #{x}}}]+ (is (= {:namespaces 6+ :modules 4+ :configured-modules 4+ :top-level-modules 3+ :nested-modules 1+ :module-edges 5+ :top-level-edges 3+ :scc-largest-modules 3+ :scc-largest-namespaces 5+ :scc-largest-pct 83.3+ :scc-cyclic-modules 3+ :scc-cyclic-namespaces 5+ :scc-components 1+ :scc-static-largest-namespaces 3+ :scc-dynamic-only-namespaces 2+ :upward-requires 1+ :upward-module-edges 1+ :upward-internal-requires 0+ :upward-cycle-requires 1+ :exposed-namespaces 3+ :hidden-modules 1+ :hidden-namespaces 1+ :cross-top-level-required-namespaces 3+ :api-any-modules 1+ :api-any-namespaces 1+ :uses-any-modules 0+ :friend-grants 1+ :cross-top-level-friend-grants 1+ :friend-exposed-namespaces 1+ :friend-access-paths 1+ :distinct-namespaces 6+ :configured-top-level-modules 3+ :scc1-modules 3+ :scc1-namespaces 5+ :scc2-modules 0+ :scc2-namespaces 0+ :scc3-modules 0+ :scc3-namespaces 0+ ;; The namespaces x.core and x.y.core require each other.+ :largest-ns-level-scc 2+ ;; Only x and z; x and x.y also require each other, but one is nested in the other.+ :mutual-dependency-pairs 1+ ;; The require from x.core to z.impl, past z's API, which z's friend grant allows.+ :friend-internal-namespaces 1+ :friend-internal-requires 1+ :non-api-cross-module-requires 1+ :total-api-namespaces 3+ :ns-prefixes 0+ :model-imports-bypass 0+ :legacy-rest-modules 0+ ;; A change to w retests only w; a change anywhere else retests the whole cycle and w.+ :retest-module-mean 86.1+ :retest-module-median 100.0+ ;; Nothing requires the namespaces x.impl and w.core, and a change to z.impl retests everything but x.impl.+ :retest-namespace-mean 47.2+ :retest-namespace-median 66.7+ ;; Nothing requires w, or the namespaces x.impl and w.core.+ :leaf-modules 1+ :leaf-module-namespaces 1+ :leaf-namespaces 2+ :retest-histogram {:module [0 1 0 0 0 0 0 0 0 5]+ :namespace [0 2 0 1 0 0 2 0 1 0]}+ ;; One cycle of 4 edges, each the only way between its ends.+ :cycles '[{:modules [x x.y z]+ :namespaces 5+ :edges 4+ :density 0.67+ :cycle-rank 2+ :strong-bridges 4}]}+ (module-history/commit-metrics deps modules)))))++(deftest ^:parallel columns-test+ (testing "every metric is a CSV column, except the two that are not a single value"+ (is (= #{:cycles :retest-histogram}+ (set/difference (set (keys (module-history/commit-metrics [] {})))+ (set @#'module-history/columns))))))++(deftest ^:parallel commit-metrics-without-cycles-test+ (testing "modules that form no cycle report zero for every cycle measure"+ (is (=? {:namespaces 2+ :modules 2+ :module-edges 1+ :top-level-edges 1+ :scc-largest-modules 0+ :scc-largest-namespaces 0+ :scc-largest-pct 0.0+ :scc-cyclic-modules 0+ :scc-cyclic-namespaces 0+ :scc-components 0+ :scc-static-largest-namespaces 0+ :scc-dynamic-only-namespaces 0+ :scc1-modules 0+ :scc1-namespaces 0+ :largest-ns-level-scc 0+ :mutual-dependency-pairs 0+ :cycles []}+ (module-history/commit-metrics+ '[{:namespace metabase.a.core+ :filename "src/metabase/a/core.clj"+ :module a+ :deps [{:namespace metabase.b.core, :module b}]}+ {:namespace metabase.b.core+ :filename "src/metabase/b/core.clj"+ :module b+ :deps []}]+ '{a {}, b {}})))))++(deftest ^:parallel commit-metrics-whole-backend-test+ (testing "before the module system the backend is one module, which has no edges and no cycle"+ (is (=? {:namespaces 2+ :modules 1+ :configured-modules 1+ :top-level-modules 1+ :nested-modules 0+ :module-edges 0+ :top-level-edges 0+ :scc-largest-pct 0.0+ :scc-components 0+ ;; Every change retests the one module.+ :retest-module-mean 100.0+ :retest-module-median 100.0+ :leaf-modules 1+ :leaf-module-namespaces 2+ :cycles []}+ (module-history/commit-metrics+ '[{:namespace metabase.lib.core+ :filename "src/metabase/lib/core.clj"+ :module backend+ :deps [{:namespace metabase.util, :module backend}]}+ {:namespace metabase.util+ :filename "src/metabase/util.clj"+ :module backend+ :deps []}]+ @#'module-history/whole-backend)))))++(def ^:private legacy-config+ (pr-str '{:linters {:metabase/ns-module-checker {:api-namespaces {metabase.lib #{metabase.lib.core}}}}}))++(deftest ^:parallel commit-modules-modern-test+ (testing "the modules config is read as it is, without the library module, even beside the older config"+ (is (= '{lib {:api #{metabase.lib.core}, :uses #{util}}}+ (#'module-history/commit-modules+ {".clj-kondo/config/modules/config.edn"+ (pr-str '{:metabase/modules {connection-pool {:api :any}+ lib {:api #{metabase.lib.core}, :uses #{util}}}})+ ".clj-kondo/config.edn"+ legacy-config})))))++(deftest ^:parallel commit-modules-legacy-test+ (testing "a commit with only the older linter config gets its modules translated"+ (is (= '{lib {:api #{metabase.lib.core}}}+ (#'module-history/commit-modules {".clj-kondo/config.edn" legacy-config})))))++(deftest ^:parallel commit-modules-whole-backend-test+ (testing "a commit with no config at all counts the backend as one module"+ (is (= @#'module-history/whole-backend+ (#'module-history/commit-modules {}))))+ (testing "so does a commit whose linter config names no modules"+ (is (= @#'module-history/whole-backend+ (#'module-history/commit-modules {".clj-kondo/config.edn" (pr-str '{:linters {}})})))))++(deftest snapshot-test+ (testing "a commit's snapshot has its module map and a dependency row for each backend namespace"+ (let [{:keys [deps modules]} (module-history/snapshot "HEAD")]+ (is (=? {:namespace 'metabase.util, :module 'util, :deps seq}+ (first (filter #(= 'metabase.util (:namespace %)) deps))))+ (is (contains? modules 'util)))))++(deftest commits-to-measure-test+ (testing "an explicit revision is measured once when the sample also picks it"+ (let [head (str/trim (#'module-history/git! ["rev-parse" "HEAD"]))+ ;; A date after every commit samples only the tip of `ref`.+ tomorrow (str (.plusDays (java.time.LocalDate/now java.time.ZoneOffset/UTC) 1))]+ (is (= [head]+ (#'module-history/commits-to-measure {:commits ["HEAD"], :ref "HEAD", :since tomorrow}))))))