The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Measure how tangled module cycles are (#83552)
Chris Truterea6218bb31c6
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex 5f7297633c2..d6f8d9ce932 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -8,8 +8,10 @@[clojure.tools.namespace.find :as ns.find][clojure.tools.namespace.parse :as ns.parse][clojure.walk :as walk]+ [dev.module-graph :as module-graph][hooks.common.modules :as modules][lambdaisland.deep-diff2 :as ddiff]+ [metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][rewrite-clj.node :as n]@@ -953,82 +955,6 @@;;;; Module boundary analysis-(defn- graph-nodes [graph]- (into (set (keys graph)) (mapcat val) graph))--(defn strongly-connected-components- "Return the strongly connected components of `graph` as a vector of sets.- A node outside every cycle comes back as a singleton set."- [graph]- ;; Tarjan's algorithm, kept recursive because that reads better than an explicit stack of frames.- ;; Each node on the search path costs a level of recursion, so the worst case is one level per node.- ;; As of 2026-09-11 the 206-module graph peaks at 40 levels; the default 2 MB thread stack fits about 2,600.- (letfn [(pop-component [state root]- (loop [state state, component #{}]- (let [node (peek (:stack state))- state (-> state- (update :stack pop)- (update :on-stack disj node))- component (conj component node)]- (if (= node root)- (update state :components conj component)- (recur state component)))))- (visit [state node]- (let [node-index (:next-index state)- state (-> state- (assoc-in [:index node] node-index)- (assoc-in [:lowlink node] node-index)- (update :next-index inc)- (update :stack conj node)- (update :on-stack conj node))- state (reduce (fn [state successor]- (cond- (not (contains? (:index state) successor))- (let [state (visit state successor)]- (update-in state [:lowlink node]- min- (get-in state [:lowlink successor])))-- (contains? (:on-stack state) successor)- (update-in state [:lowlink node]- min- (get-in state [:index successor]))-- :else- state))- state- (get graph node))]- (cond-> state- (= (get-in state [:lowlink node]) (get-in state [:index node]))- (pop-component node))))]- (:components- (reduce (fn [state node]- (cond-> state- (not (contains? (:index state) node)) (visit node)))- {:components []- :index {}- :lowlink {}- :next-index 0- :on-stack #{}- :stack []}- (sort (graph-nodes graph))))))--(defn cyclic-components- "Non-singleton [[strongly-connected-components]], largest first; ties sort by first member."- [graph]- (->> (strongly-connected-components graph)- (filter #(> (count %) 1))- (sort-by (fn [component] [(- (count component)) (str (first (sort component)))]))- vec))--(defn- cyclic-component-sizes- "Module and namespace counts for each [[cyclic-components]] cluster."- [graph node->namespace-count]- (mapv (fn [component]- {:modules (count component)- :namespaces (transduce (map #(get node->namespace-count % 0)) + 0 component)})- (cyclic-components graph)))-(defn module-boundary-stats"REPL diagnostics for the module graph:@@ -1036,19 +962,36 @@- `:module-count` configured modules- `:scc-module-sizes` modules per cycle, largest first- `:scc-namespace-sizes` namespaces per cycle, in the same order+ - `:scc-densities` edges per cycle over the most it could have, in the same order+ - `:scc-cycle-ranks` independent cycles per cycle, in the same orderThese values are not ratcheted because any source change can move them. Use namespace sizes to track- cycle reduction: splitting a module can grow a cycle's module count without removing namespaces."+ cycle reduction: splitting a module can grow a cycle's module count without removing namespaces.+ Density compares how tangled cycles of different sizes are.+ Cycle rank is an upper bound on the edges to cut."([](module-boundary-stats (dependencies) (kondo-config)))([deps config](let [any-modules (into #{} (keep (fn [[module cfg]] (when (= :any (:api cfg)) module))) config)- sizes (cyclic-component-sizes (module-dependencies deps)- (frequencies (keep :module deps)))]+ ns-counts (frequencies (keep :module deps))+ cycles (module-graph/cycles (module-dependencies deps))]{:api-any-namespaces (count (filter #(contains? any-modules (:module %)) deps)):module-count (count config)- :scc-module-sizes (mapv :modules sizes)- :scc-namespace-sizes (mapv :namespaces sizes)})))+ :scc-module-sizes (mapv :size cycles)+ :scc-namespace-sizes (mapv (fn [{:keys [modules]}]+ (transduce (map #(get ns-counts % 0)) + 0 modules))+ cycles)+ :scc-densities (mapv #(u/round-to-decimals 2 (:density %)) cycles)+ :scc-cycle-ranks (mapv :cycle-rank cycles)})))++(defn module-cycles+ "Each module cycle with its members and [[module-graph/cycle-metrics]], largest first.+ The strong bridges are the single dependency edges whose removal splits a cycle."+ ([]+ (module-cycles (dependencies)))+ ([deps]+ (module-graph/cycles (module-dependencies deps))))(comment- (module-boundary-stats))+ (module-boundary-stats)+ (module-cycles))diff --git a/dev/src/dev/module_cycle_names.clj b/dev/src/dev/module_cycle_names.cljnew file mode 100644index 00000000000..058b36ea000--- /dev/null+++ b/dev/src/dev/module_cycle_names.clj@@ -0,0 +1,55 @@+(ns dev.module-cycle-names+ "Reads and validates the file that names the module require graph's cyclic clusters.+ Plain Clojure, so the Babashka-run module explorer can show the names too."+ (:require+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.string :as str]))++(set! *warn-on-reflection* true)++(def clusters-file+ "The file of cluster names, relative to the repo root."+ ".clj-kondo/config/modules/cycle-clusters.edn")++(defn- read-form+ "The one EDN form in the file at `path`. Throws when the file is empty or holds a second form."+ [path]+ (with-open [reader (java.io.PushbackReader. (io/reader path))]+ (let [eof (Object.)+ form (edn/read {:eof eof} reader)]+ (when (identical? eof form)+ (throw (ex-info (str path " is empty; expected a map of cluster name to anchor module")+ {:file path})))+ (when-not (identical? eof (edn/read {:eof eof} reader))+ (throw (ex-info (str path " holds more than one form; expected one map")+ {:file path})))+ form)))++(defn validate-anchors+ "Return `anchors` if it maps simple-symbol names to module symbols, no module anchoring two; throw otherwise."+ [anchors]+ (when-not (map? anchors)+ (throw (ex-info (str clusters-file " must hold a map of cluster name to anchor module, not " (pr-str anchors))+ {:anchors anchors})))+ (doseq [[cluster-name anchor] anchors]+ (when-not (simple-symbol? cluster-name)+ (throw (ex-info (format "%s is not a cluster name; names are simple symbols" (pr-str cluster-name))+ {:cluster cluster-name})))+ (when-not (symbol? anchor)+ (throw (ex-info (format "%s must be anchored by a module symbol, not %s" cluster-name (pr-str anchor))+ {:cluster cluster-name, :anchor anchor}))))+ (doseq [[anchor names] (group-by val anchors)+ :when (< 1 (count names))]+ (throw (ex-info (format "%s anchors more than one cluster: %s" anchor (str/join ", " (sort (map key names))))+ {:anchor anchor, :clusters (sort (map key names))})))+ anchors)++(defn read-anchors+ "Parsed and validated contents of the file at `path`, [[clusters-file]] by default."+ ([]+ (read-anchors clusters-file))+ ([path]+ (when-not (.exists (io/file path))+ (throw (ex-info (str path " is missing") {:file path})))+ (validate-anchors (read-form path))))diff --git a/dev/src/dev/module_cycles.clj b/dev/src/dev/module_cycles.cljindex 07a247e5d8b..fc68bf729e3 100644--- a/dev/src/dev/module_cycles.clj+++ b/dev/src/dev/module_cycles.clj@@ -1,72 +1,23 @@(ns dev.module-cycles"Named cyclic clusters of the module require graph.- The file at [[clusters-file]] names each cluster by an anchor module: the cluster holding it carries the name.+ The file at [[cycle-names/clusters-file]] names each cluster by an anchor module.+ The cluster holding the anchor carries the name.Membership is computed, never recorded, so a cluster can grow or shrink without an edit.Every cluster must hold exactly one anchor.Two anchors in one cluster mean two named clusters merged.A cluster without one is a new cycle or the unnamed half of a split.An anchor in no cluster names a cycle that is gone."(:require- [clojure.edn :as edn]- [clojure.java.io :as io][clojure.string :as str][dev.deps-graph :as deps-graph]+ [dev.module-cycle-names :as cycle-names]+ [dev.module-graph :as module-graph][hooks.common.modules :as modules][metabase.util :as u]))(set! *warn-on-reflection* true)-(def clusters-file- "The file of cluster names, relative to the repo root."- ".clj-kondo/config/modules/cycle-clusters.edn")--;;;; =============================================================================-;;;; Reading the names-;;;; =============================================================================--(defn- read-form- "The one EDN form in the file at `path`. Throws when the file is empty or holds a second form."- [path]- (with-open [reader (java.io.PushbackReader. (io/reader path))]- (let [eof (Object.)- form (edn/read {:eof eof} reader)]- (when (identical? eof form)- (throw (ex-info (str path " is empty; expected a map of cluster name to anchor module")- {:file path})))- (when-not (identical? eof (edn/read {:eof eof} reader))- (throw (ex-info (str path " holds more than one form; expected one map")- {:file path})))- form)))--(defn validate-anchors- "Return `anchors` if it maps simple-symbol names to module symbols, no module anchoring two; throw otherwise."- [anchors]- (when-not (map? anchors)- (throw (ex-info (str clusters-file " must hold a map of cluster name to anchor module, not " (pr-str anchors))- {:anchors anchors})))- (doseq [[cluster-name anchor] anchors]- (when-not (simple-symbol? cluster-name)- (throw (ex-info (format "%s is not a cluster name; names are simple symbols" (pr-str cluster-name))- {:cluster cluster-name})))- (when-not (symbol? anchor)- (throw (ex-info (format "%s must be anchored by a module symbol, not %s" cluster-name (pr-str anchor))- {:cluster cluster-name, :anchor anchor}))))- (doseq [[anchor names] (group-by val anchors)- :when (< 1 (count names))]- (throw (ex-info (format "%s anchors more than one cluster: %s" anchor (str/join ", " (sort (map key names))))- {:anchor anchor, :clusters (sort (map key names))})))- anchors)--(defn read-anchors- "Parsed and validated contents of the file at `path`, [[clusters-file]] by default."- ([]- (read-anchors clusters-file))- ([path]- (when-not (.exists (io/file path))- (throw (ex-info (str path " is missing") {:file path})))- (validate-anchors (read-form path))))-;;;; =============================================================================;;;; Finding problems;;;; =============================================================================@@ -154,7 +105,7 @@- `:anchor-left`, a name whose anchor is in no cycle: `:name`, `:anchor`- `:undeclared-anchor`, a name anchored on a module not among `modules`: `:name`, `:anchor`"[graph modules anchors]- (let [components (deps-graph/cyclic-components graph)+ (let [components (module-graph/cyclic-components graph)clusters (map (juxt identity (names-in modules anchors)) components)in-any (into #{} cat components)undeclared (into #{} (remove modules) (vals anchors))@@ -212,7 +163,7 @@(str " If instead only an anchor moved, and the cluster it left is still a cycle, anchor that name on a module"" still in it.")(str " Only if there is truly no way around it: remove all but one of the names from "- clusters-file " and explain why in the PR.")]))+ cycle-names/clusters-file " and explain why in the PR.")]))(defn- unnamed-lines [{:keys [cluster requires proposal]}](concat@@ -225,7 +176,7 @@" the name-module-cycle skill in .claude/skills/name-module-cycle.")(if-let [{:keys [name anchor]} proposal](format " Then add a line like `%s %s` to %s. Any declared member can be the anchor." name anchor- clusters-file)+ cycle-names/clusters-file)(str " None of its modules is declared in .clj-kondo/config/modules/config.edn yet. Add an entry with a :team"" for each, run `./bin/mage fix-modules-config` to fill in the rest, then anchor the name on one of them."))" If this is a brand new cycle instead, break it rather than naming it."]))@@ -234,12 +185,13 @@[(format "%s is anchored on %s, which is no longer in any cycle." cluster-name anchor)(str " If the rest of the cluster is still a cycle, reported above without a name, keep the name and anchor it"" on one of its modules instead.")- (format " Otherwise the cycle is gone. Nice work. Remove its line from %s to retire the name." clusters-file)])+ (format " Otherwise the cycle is gone. Nice work. Remove its line from %s to retire the name."+ cycle-names/clusters-file)])(defn- undeclared-anchor-lines [{cluster-name :name, :keys [anchor]}][(format (str "%s is anchored on %s, which is not a declared module. Declare it in"" .clj-kondo/config/modules/config.edn, or anchor the name on a declared module in %s.")- cluster-name anchor clusters-file)])+ cluster-name anchor cycle-names/clusters-file)])(defn message"The failure message for one of the [[problems]]."@@ -262,10 +214,10 @@{:config config:graph (deps-graph/module-dependencies (deps-graph/dependencies (modules/build-prefix->module config))):modules (set (keys config))- :anchors (read-anchors)}))+ :anchors (cycle-names/read-anchors)}))(defn report- "The failure messages for the [[problems]] between the current tree's module require graph and [[clusters-file]]."+ "The failure messages for the [[problems]] between the current tree's module graph and [[cycle-names/clusters-file]]."[](let [{:keys [graph modules anchors]} (repository)](mapv message (problems graph modules anchors))))@@ -277,7 +229,7 @@(let [{:keys [config graph modules anchors]} (repository)named-in (names-in modules anchors);; Unnamed clusters first: they are the ones someone is here to name.- clusters (sort-by #(boolean (seq (named-in %))) (deps-graph/cyclic-components graph))+ clusters (sort-by #(boolean (seq (named-in %))) (module-graph/cyclic-components graph))placeholders (proposals graph modules anchors (remove (comp seq named-in) clusters))](doseq [cluster clusters:let [named (named-in cluster)]]diff --git a/dev/src/dev/module_graph.clj b/dev/src/dev/module_graph.cljnew file mode 100644index 00000000000..5f949f9c33c--- /dev/null+++ b/dev/src/dev/module_graph.clj@@ -0,0 +1,121 @@+(ns dev.module-graph+ "Cycle analysis for a dependency graph given as `node -> #{successor}`.++ Plain Clojure only: `dev.deps-graph` uses it in a dev REPL, and `dev.module-explorer` under Babashka.")++(defn- graph-nodes [graph]+ (into (set (keys graph)) (mapcat val) graph))++(defn strongly-connected-components+ "Return the strongly connected components of `graph` as a vector of sets.+ A node outside every cycle comes back as a singleton set."+ [graph]+ ;; Tarjan's algorithm, kept recursive because that reads better than an explicit stack of frames.+ ;; Each node on the search path costs a level of recursion, so the worst case is one level per node.+ ;; As of 2026-09-11 the 206-module graph peaks at 40 levels; the default 2 MB thread stack fits about 2,600.+ (letfn [(pop-component [state root]+ (loop [state state, component #{}]+ (let [node (peek (:stack state))+ state (-> state+ (update :stack pop)+ (update :on-stack disj node))+ component (conj component node)]+ (if (= node root)+ (update state :components conj component)+ (recur state component)))))+ (visit [state node]+ (let [node-index (:next-index state)+ state (-> state+ (assoc-in [:index node] node-index)+ (assoc-in [:lowlink node] node-index)+ (update :next-index inc)+ (update :stack conj node)+ (update :on-stack conj node))+ state (reduce (fn [state successor]+ (cond+ (not (contains? (:index state) successor))+ (let [state (visit state successor)]+ (update-in state [:lowlink node]+ min+ (get-in state [:lowlink successor])))++ (contains? (:on-stack state) successor)+ (update-in state [:lowlink node]+ min+ (get-in state [:index successor]))++ :else+ state))+ state+ (get graph node))]+ (cond-> state+ (= (get-in state [:lowlink node]) (get-in state [:index node]))+ (pop-component node))))]+ (:components+ (reduce (fn [state node]+ (cond-> state+ (not (contains? (:index state) node)) (visit node)))+ {:components []+ :index {}+ :lowlink {}+ :next-index 0+ :on-stack #{}+ :stack []}+ (sort (graph-nodes graph))))))++(defn cyclic-components+ "Non-singleton [[strongly-connected-components]], largest first; ties sort by first member."+ [graph]+ (->> (strongly-connected-components graph)+ (filter #(> (count %) 1))+ (sort-by (fn [component] [(- (count component)) (first (sort component))]))+ vec))++(defn- component-edges+ "The `[from to]` edges of `graph` with both ends in `component`, sorted."+ [graph component]+ (vec (sort (for [from component+ to (get graph from)+ :when (and (not= from to) (contains? component to))]+ [from to]))))++(defn- strong-bridges+ "The `edges` of a strongly connected component whose removal leaves it no longer strongly connected."+ [edges]+ ;; Removing `a -> b` breaks the component exactly when `b` is no longer reachable from `a`:+ ;; every other path that used the edge can detour through the remaining route from `a` to `b`.+ (let [successors (reduce (fn [acc [from to]] (update acc from (fnil conj []) to)) {} edges)+ reachable? (fn [[from to :as skipped]]+ (loop [stack [from], seen #{from}]+ (if-let [node (peek stack)]+ (if (= node to)+ true+ (let [next-nodes (remove #(or (seen %) (= [node %] skipped)) (successors node))]+ (recur (into (pop stack) next-nodes) (into seen next-nodes))))+ false)))]+ (filterv (complement reachable?) edges)))++(defn cycle-metrics+ "How tangled a strongly connected `component` of `graph` is.++ - `:size` nodes in the component+ - `:edges` edges inside it+ - `:density` edges over the n(n-1) possible, so 1.0 means every pair depends on each other+ - `:cycle-rank` edges - nodes + 1: independent cycles, and at most the edges to cut to break them all+ - `:strong-bridges` sorted edges whose removal leaves the component no longer strongly connected"+ [graph component]+ (let [n (count component)+ edges (component-edges graph component)+ m (count edges)]+ {:size n+ :edges m+ :density (if (> n 1) (double (/ m (* n (dec n)))) 0.0)+ :cycle-rank (if (> n 1) (inc (- m n)) 0)+ :strong-bridges (strong-bridges edges)}))++(defn cycles+ "Each cycle in `graph`, largest first: its sorted `:modules` plus its [[cycle-metrics]]."+ [graph]+ (mapv (fn [component]+ (assoc (cycle-metrics graph component) :modules (vec (sort component))))+ (cyclic-components graph)))diff --git a/dev/test/metabase/core/module_cycles_test.clj b/dev/test/metabase/core/module_cycles_test.cljindex 070aef27186..f0a7fca3378 100644--- a/dev/test/metabase/core/module_cycles_test.clj+++ b/dev/test/metabase/core/module_cycles_test.clj@@ -4,6 +4,7 @@[clojure.string :as str][clojure.test :refer :all][dev.kondo-ratchet :as kondo-ratchet]+ [dev.module-cycle-names :as cycle-names][dev.module-cycles :as module-cycles]))(set! *warn-on-reflection* true)@@ -97,7 +98,7 @@(deftest ^:parallel names-file-test(testing "malformed names throw"- (are [anchors msg] (thrown-with-msg? clojure.lang.ExceptionInfo msg (module-cycles/validate-anchors anchors))+ (are [anchors msg] (thrown-with-msg? clojure.lang.ExceptionInfo msg (cycle-names/validate-anchors anchors))'[foundation app-db] #"must hold a map"'{:foundation app-db} #"not a cluster name"'{foundation "app-db"} #"must be anchored by a module symbol"@@ -107,14 +108,14 @@(try(are [contents msg] (thrown-with-msg? clojure.lang.ExceptionInfo msg(do (spit file contents)- (module-cycles/read-anchors (.getPath file))))+ (cycle-names/read-anchors (.getPath file))))";; nothing\n" #"is empty""{a app-db}\n{b qp}\n" #"more than one form")(finally(io/delete-file file true)))))(testing "a missing file throws"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"is missing"- (module-cycles/read-anchors "no/such/cycle-clusters.edn")))))+ (cycle-names/read-anchors "no/such/cycle-clusters.edn")))))(deftest ^:parallel messages-test(testing "a merge names the clusters, shows the chains, and keeps removing a name as the last resort"@@ -169,4 +170,4 @@(let [problems (module-cycles/report)](is (empty? problems)(str "Every cyclic cluster of the module require graph holds exactly one anchor from "- module-cycles/clusters-file ".\n\n" (str/join "\n\n" problems))))))+ cycle-names/clusters-file ".\n\n" (str/join "\n\n" problems))))))diff --git a/dev/test/metabase/core/modules_test.clj b/dev/test/metabase/core/modules_test.cljindex 6ad7576c0f2..fcd5132c7d7 100644--- a/dev/test/metabase/core/modules_test.clj+++ b/dev/test/metabase/core/modules_test.clj@@ -7,6 +7,7 @@[clojure.test :refer :all][dev.deps-graph][dev.model-boundary-config]+ [dev.module-graph :as module-graph][hooks.common.modules :as modules][metabase.test-runner][metabase.util.json :as json]@@ -358,9 +359,50 @@:f #{:d}:g #{:h}}](is (= #{#{:a :b} #{:c} #{:d :e :f} #{:g} #{:h}}- (set (dev.deps-graph/strongly-connected-components graph))))+ (set (module-graph/strongly-connected-components graph))))(is (= [#{:d :e :f} #{:a :b}]- (dev.deps-graph/cyclic-components graph)))))+ (module-graph/cyclic-components graph)))))++(deftest ^:parallel cycle-metrics-test+ (testing "a ring is one independent cycle, and every edge holds it together"+ (is (= {:size 4+ :edges 4+ :density (double (/ 4 12))+ :cycle-rank 1+ :strong-bridges [[:a :b] [:b :c] [:c :d] [:d :a]]}+ (module-graph/cycle-metrics {:a #{:b} :b #{:c} :c #{:d} :d #{:a}} #{:a :b :c :d}))))+ (testing "a complete digraph has every edge and no single edge that splits it"+ (is (= {:size 3+ :edges 6+ :density 1.0+ :cycle-rank 4+ :strong-bridges []}+ (module-graph/cycle-metrics {:a #{:b :c} :b #{:a :c} :c #{:a :b}} #{:a :b :c}))))+ (testing "a chord across a ring is not a strong bridge, while every ring edge still is"+ (is (=? {:cycle-rank 2, :strong-bridges [[:a :b] [:b :c] [:c :a]]}+ (module-graph/cycle-metrics {:a #{:b :c} :b #{:c} :c #{:a}} #{:a :b :c}))))+ (testing "edges leaving the component and self-loops are not counted"+ (is (=? {:edges 4, :strong-bridges [[:a :b] [:b :a] [:b :c] [:c :b]]}+ (module-graph/cycle-metrics {:a #{:a :b :x} :b #{:a :c} :c #{:b}} #{:a :b :c})))))++(deftest ^:parallel module-cycle-stats-test+ (let [deps [{:module 'a, :deps [{:module 'b}]}+ {:module 'a, :deps [{:module 'c}]}+ {:module 'b, :deps [{:module 'c}]}+ {:module 'c, :deps [{:module 'a}]}+ {:module 'd, :deps [{:module 'a}]}]+ config '{a {:api :any}+ b {}+ c {}+ d {}}]+ (testing "stats list each cycle's sizes, rounded density, and cycle rank"+ (is (= {:api-any-namespaces 2+ :module-count 4+ :scc-module-sizes [3]+ :scc-namespace-sizes [4]+ :scc-densities [0.67]+ :scc-cycle-ranks [2]}+ (dev.deps-graph/module-boundary-stats deps config))))))(deftest ^:parallel module-boundary-config-values-have-valid-types-test(testing "Module boundary keys have the shapes the ratchet counts expect"