Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add `mage module-history` to update and publish the module history (#84122)

Chris Truterb6d65f612d5c
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 12907f7fd8d..1edc2e678cb 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -1203,7 +1203,11 @@
mage-namespaces
{:linters
- {:discouraged-java-method
+ {;; mage runs under babashka, which bundles cheshire; metabase.util.json is not on its classpath.
+ :discouraged-namespace
+ {cheshire.core {:level :off}}
+
+ :discouraged-java-method
{java.lang.System {exit
{:level :error
:message "Don't use System/exit directly. Instead use mage.util/exit and let mage handle exiting the process"}}}}}}}
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index ee8c8ee3696..634e396b87f 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -85,7 +85,7 @@
:config-counts {:constant-condition 1
:deprecated-namespace 7
:deprecated-var 16
- :discouraged-namespace 14
+ :discouraged-namespace 15
:discouraged-var 14
:inline-def 1
:main-without-gen-class 1
diff --git a/bb.edn b/bb.edn
index c30ea1d4c2d..26d9a220c62 100644
--- a/bb.edn
+++ b/bb.edn
@@ -336,6 +336,22 @@
:requires [[mage.modules]]
:task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}
+ module-history
+ {:doc "Update the CSV files that track how the backend module graph changed on master, and optionally publish them. Measures only commits it has not measured before."
+ :examples [["./bin/mage module-history --dir ~/module-history" "Fetch master, measure its new commits, and write the CSV files and history.json"]
+ ["./bin/mage module-history --dir ~/module-history --publish ~/module-history/publish.edn" "Then fill pages, replace Metabase uploads, refresh cards and update shares, as the EDN file says"]
+ ["./bin/mage module-history --dir ~/module-history-ahead --ref origin/my-stack-tip" "Write the history as it would be with a branch merged: master up to where the branch left it, then the branch's commits"]
+ ["./bin/mage module-history --dir ~/module-history --no-measure --publish ~/module-history/publish.edn --dry-run" "Print what publishing the files already there would run"]]
+ :options [["-d" "--dir DIR" "Directory to keep the files in (required)"]
+ ["-P" "--publish FILE" "Publish the files as this EDN file says; see the mage.module-history docstring for its shape"]
+ ["-D" "--dry-run" "With --publish, print the commands that would change Metabase or a share instead of running them"]
+ ["-r" "--ref REF" "Measure the first-parent history of this ref instead of origin/master"]
+ ["-F" "--no-fetch" "Measure the ref as last fetched"]
+ ["-M" "--no-measure" "Skip fetching and measuring; use the files already in DIR"]]
+ :arg-schema [:tuple]
+ :requires [[mage.module-history]]
+ :task (task! (mage.module-history/cli-update parsed))}
+
fix-modules-config
{:doc "Regenerate .clj-kondo/config/modules/config.edn so it passes metabase.core.modules-test."
:examples [["./bin/mage fix-modules-config" "Fix the modules config (uses the running dev REPL if available, else a JVM)"]
diff --git a/dev/src/dev/module_cuts.clj b/dev/src/dev/module_cuts.clj
new file mode 100644
index 00000000000..223b4d79f49
--- /dev/null
+++ b/dev/src/dev/module_cuts.clj
@@ -0,0 +1,131 @@
+(ns dev.module-cuts
+ "Where the module graph could be cut next, from one commit's dependency rows.
+
+ [[dev.module-history-report]] writes both tables for the latest commit it measures."
+ (:require
+ [dev.deps-graph :as deps-graph]
+ [dev.module-graph :as module-graph]))
+
+(set! *warn-on-reflection* true)
+
+(defn upward-pairs
+ "Module and ancestor pairs whose upward requires close a cycle, the pair with the most requires first.
+ Each is `{:module, :ancestor, :requires, :internal}`: how many requires go from the module into that ancestor,
+ and how many of those reach past the ancestor's `:api`."
+ [deps modules]
+ (->> (deps-graph/upward-references deps modules)
+ (filter :cycle?)
+ (group-by (juxt :module :ancestor))
+ (map (fn [[[module ancestor] requires]]
+ {:module module
+ :ancestor ancestor
+ :requires (count requires)
+ :internal (count (remove :api? requires))}))
+ (sort-by (juxt (comp - :requires) (comp str :module) (comp str :ancestor)))
+ vec))
+
+;;;; Cut candidates
+
+(defn- requires-by-edge
+ "The namespace requires behind each module edge: `{[from to] [[from-namespace to-namespace] ...]}`."
+ [deps]
+ (reduce (fn [edges [edge require]]
+ (update edges edge (fnil conj []) require))
+ {}
+ (for [{:keys [namespace module], requires :deps} deps
+ {to-namespace :namespace, to :module} requires
+ :when (and module to (not= module to))]
+ [[module to] [namespace to-namespace]])))
+
+(defn- largest-cycle
+ "The modules of the cycle of `graph` that holds the most namespaces, or nil without a cycle."
+ [graph size]
+ (when-let [cycles (seq (module-graph/cyclic-components graph))]
+ (apply max-key size cycles)))
+
+(defn- bottom-up
+ "The modules of `graph`, one cycle, ordered so that few requires point from an earlier module to a later one.
+ That is roughly the cycle as layers, the most depended-upon first.
+ The `weight` of an edge is the number of requires behind it."
+ [graph weight]
+ ;; Eades, Lin and Smyth's greedy heuristic for a small feedback arc set: take modules that require nothing left
+ ;; to the bottom and modules nothing left requires to the top, and otherwise the one that gives up the fewest
+ ;; requires by going to the bottom.
+ (let [requirers (reduce (fn [requirers [from to]] (update requirers to (fnil conj #{}) from))
+ {}
+ (for [[from tos] graph, to tos] [from to]))
+ total (fn [edges] (transduce (map weight) + 0 edges))]
+ (loop [left (set (keys graph))
+ bottom []
+ top ()]
+ (if (empty? left)
+ (vec (concat bottom top))
+ (let [out (fn [module] (total (for [to (graph module) :when (left to)] [module to])))
+ in (fn [module] (total (for [from (requirers module) :when (left from)] [from module])))
+ sinks (filter #(zero? (out %)) left)
+ sources (filter #(zero? (in %)) left)]
+ (cond
+ (seq sinks) (recur (apply disj left sinks) (into bottom (sort-by str sinks)) top)
+ (seq sources) (recur (apply disj left sources) bottom (concat (sort-by str sources) top))
+ :else (let [module (apply min-key #(- (out %) (in %)) (sort-by str left))]
+ (recur (disj left module) (conj bottom module) top))))))))
+
+(defn cut-candidates
+ "Ways to shrink the largest module cycle by splitting it into a lower and an upper layer.
+ Each candidate takes the bottom `:base-modules` of the cycle's `:cycle-modules`, in [[bottom-up]] order, as the
+ lower layer and severs every require from it into the rest of the cycle:
+
+ {:base-modules 44
+ :cycle-modules 53
+ :requires [[from-namespace to-namespace] ...]
+ :edges [[from-module to-module requires] ...]
+ :largest-after 681
+ :freed 113}
+
+ The `:edges` are the module edges behind `:requires`, the one with the most requires first.
+ The `:largest-after` count is the namespaces in the largest cycle left among those modules once the requires
+ are gone, and `:freed` how many fewer that is.
+ Other cycles of the graph are not looked at.
+ Returns the candidates of at most `max-requires` requires that no other beats on both counts, fewest requires first.
+ Empty when there is no cycle."
+ [deps max-requires]
+ (let [requires (requires-by-edge deps)
+ counts (frequencies (keep :module deps))
+ size (fn [modules] (transduce (map #(counts % 0)) + 0 modules))
+ graph (reduce (fn [graph [from to]] (update graph from (fnil conj #{}) to))
+ (zipmap (keys counts) (repeat #{}))
+ (keys requires))
+ cycle (largest-cycle graph size)]
+ (if-not cycle
+ []
+ (let [inner (into {} (map (fn [module] [module (into #{} (filter cycle) (graph module))])) cycle)
+ before (size cycle)
+ order (bottom-up inner #(count (requires %)))]
+ (->> (for [n (range 1 (count order))
+ :let [base (set (take n order))
+ severed (sort (for [from base
+ to (inner from)
+ :when (not (base to))]
+ [from to]))
+ cut (into [] (mapcat #(sort (requires %))) severed)]
+ :when (<= (count cut) max-requires)
+ :let [after (size (largest-cycle (reduce (fn [graph [from to]] (update graph from disj to))
+ inner
+ severed)
+ size))]]
+ {:base-modules n
+ :cycle-modules (count order)
+ :requires cut
+ :edges (->> severed
+ (map (fn [edge] (conj edge (count (requires edge)))))
+ (sort-by (juxt (comp - peek) first second))
+ vec)
+ :largest-after after
+ :freed (- before after)})
+ (sort-by (juxt (comp count :requires) (comp - :freed)))
+ ;; Keep a candidate only when it frees more than every cheaper one.
+ (reduce (fn [kept candidate]
+ (if (> (:freed candidate) (:freed (peek kept) 0))
+ (conj kept candidate)
+ kept))
+ []))))))
diff --git a/dev/src/dev/module_history.clj b/dev/src/dev/module_history.clj
index ae700c7ffe8..35fd180d104 100644
--- a/dev/src/dev/module_history.clj
+++ b/dev/src/dev/module_history.clj
@@ -554,12 +554,13 @@
(defn- config-commits
"Commits of `ref` since `date` that changed the module config.
Before [[modules-config-path]] existed that is a change to [[legacy-config-path]], which is followed only
- up to the commit that moved the config out of it, or to `ref` when `ref` predates that commit."
+ up to the commit that moved the config out of it, or to `ref` when `ref` predates that commit.
+ The walk starts from that commit, which is itself a config change: [[sample-commits]] removes the duplicate."
[ref date]
(let [introduced (last (str/split-lines (git! ["log" "--first-parent" "--diff-filter=A" "--format=%H" ref
"--" modules-config-path])))]
(concat (commits-since ref date [modules-config-path])
- (commits-since (if (str/blank? introduced) ref (str introduced "^")) date [legacy-config-path]))))
+ (commits-since (if (str/blank? introduced) ref introduced) date [legacy-config-path]))))
(defn- last-commit-before
"The last first-parent commit of `ref` before each of `dates`."
@@ -573,6 +574,14 @@
(let [today (LocalDate/now ZoneOffset/UTC)]
(take-while #(not (.isAfter ^LocalDate % today)) (iterate step start))))
+(defn require-full-history!
+ "Throw when the repo is a shallow clone. Its history stops at the clone's boundary, so a sample taken there would
+ miss commits, or start from the wrong one, without saying so."
+ []
+ (when (= "true" (str/trim (git! ["rev-parse" "--is-shallow-repository"])))
+ (throw (ex-info "Sampling needs the full git history, but this is a shallow clone: run `git fetch --unshallow`"
+ {}))))
+
(defn sample-commits
"Commits to measure on the first-parent history of `ref` (default `origin/master`), oldest first:
@@ -583,6 +592,7 @@
Dates are days in UTC."
[{:keys [ref since backend-since weekly-since monthly-since extra], :or {ref "origin/master"}}]
+ (require-full-history!)
(let [changed (concat (when since (config-commits ref since))
(when backend-since (concat (commits-since ref backend-since source-pathspecs)
(config-commits ref backend-since))))
diff --git a/dev/src/dev/module_history_report.clj b/dev/src/dev/module_history_report.clj
new file mode 100644
index 00000000000..2f7f1277671
--- /dev/null
+++ b/dev/src/dev/module_history_report.clj
@@ -0,0 +1,584 @@
+(ns dev.module-history-report
+ "The history of the backend module graph as CSV files, for charting how its cycles and boundaries changed.
+
+ [[update!]] measures a fixed sample of master commits with [[dev.module-history]] and writes these files:
+
+ - `history.csv`: one row per sampled commit since the first one, with cycle, exposure, `:friends` and
+ upward-require counts
+ - `structure.csv`: every recent backend-touching commit, thinned to the ends of each run of unchanged metrics
+ - `cycles.csv`: one row per commit and module cycle, named after the cluster it is today
+ - `history.json`: the commits of `history.csv` with each named cluster's size, for a page that charts them
+ - `upward-pairs.csv` and `cuts.csv`: at the latest commit, the module pairs whose upward requires close a cycle,
+ and the cheapest ways to split the largest cycle; see [[dev.module-cuts]]. Both are in `history.json` too.
+
+ The sample depends only on the history of `origin/master`, so two runs at the same commit write the same files.
+ A later run adds rows and keeps the earlier ones, except the tip of an earlier run, which stays only when the
+ sample picks it for another reason. `./bin/mage module-history` runs it.
+
+ This is a separate namespace because the source of [[dev.module-history]] is part of its cache key: a change to
+ how rows are reported should not measure every commit again."
+ (:require
+ [clojure.data.csv :as csv]
+ [clojure.edn :as edn]
+ [clojure.java.io :as io]
+ [clojure.java.shell :as sh]
+ [clojure.string :as str]
+ [dev.module-cuts :as module-cuts]
+ [dev.module-cycle-names :as cycle-names]
+ [dev.module-history :as module-history]
+ [metabase.util.json :as json])
+ (:import
+ (java.io File)
+ (java.nio.file CopyOption Files StandardCopyOption)
+ (java.time Instant LocalDate OffsetDateTime)
+ (java.util Locale)))
+
+(set! *warn-on-reflection* true)
+
+;;;; The sample
+
+(def ^:private default-ref "origin/master")
+
+(def ^:private history-sample
+ "Commits in `history.csv`, besides the [[quarterly-commits]] before it: one a month from before the first module
+ linter (#42250), one a week since that linter, and every commit that changed the module config since the dense
+ sample starts."
+ {:monthly-since "2023-12-01"
+ :weekly-since "2024-05-08"
+ :since "2025-10-01"})
+
+(def ^:private first-quarter
+ "The first quarter [[quarterly-commits]] takes the last commit before."
+ "2015-04-01")
+
+(def ^:private structure-sample
+ "Commits in `structure.csv`: every one that touched the backend since the structure dashboard was first built."
+ {:backend-since "2026-08-08"})
+
+(def key-commits
+ "Commits that changed the module system or the shape of the graph, by short sha. Each is measured along with its
+ parent, so the step it made shows, and labelled in `history.csv`."
+ {"1e9e60a0fb4" "#42250 Module linter 1.0"
+ "f88dafb8be2" "#53583 Module linter 2.0"
+ "734949f2676" "#65236 Halloween decoupling"
+ "94d2daaafd9" "#65681 Detangling pre-Conj"
+ "30616565b45" "#65144 OpenAPI→TS (regression)"
+ "119487b1ccc" "#67650 Fix api→api-routes"
+ "ee9ba657624" "#67292 :friends introduced"
+ "c2ad36cccbd" "#71068 ee/llm into OSS"
+ "ff335470e72" "#81824 Per-module db.clj"
+ "eff3209a34b" "#82301 Module hierarchy"
+ "4f02379e40d" "#82367 Domain subtrees"
+ "30346d6a64a" "#82011 Break requiring-resolve cycles"
+ "f35a8a0077b" "#77954 Untangle util"
+ "ca08e86675b" "#83441 Decouple foundation modules"
+ "06b78789e31" "#83453 Feature modules out of the cycle"
+ "e902d2623c7" "#82386 Name module cycles"})
+
+(def ^:private gone-clusters
+ "Names for cycles that were gone before `cycle-clusters.edn` existed, so they have no anchor there: cluster name
+ and a module only that cycle held."
+ ;; magellanic-cloud: segments and xrays drifted beside galactic-center from #65681 until #82447 pulled them back in,
+ ;; as the Magellanic Clouds are being pulled into the Milky Way.
+ ;; sub-etha: the EE Metabot and Slackbot cycle, Metabot talking over Slack, named after the Hitchhiker's messaging
+ ;; network to sit beside deep-thought. It lasted six days, until #71068 moved ee/llm into OSS.
+ '[[magellanic-cloud xrays]
+ [sub-etha enterprise/slackbot]])
+
+;;;; Git
+
+(defn- git-lines
+ "Output lines of git run with `args`, throwing when it fails."
+ [args]
+ (let [{:keys [exit out err]} (apply sh/sh "git" args)]
+ (when-not (zero? exit)
+ (throw (ex-info (format "git %s failed: %s" (str/join " " args) err) {:args args})))
+ (remove str/blank? (str/split-lines out))))
+
+(defn- positions
+ "First-parent position of every commit of `ref`, oldest first, by sha. This is `seq` in the files."
+ ;; Commit timestamps carry mixed UTC offsets and two commits can share one, so rows are ordered by this and never
+ ;; by timestamp.
+ [ref]
+ (zipmap (git-lines ["rev-list" "--first-parent" "--reverse" ref])
+ (range)))
+
+(defn- quarter-starts
+ "The first day of each quarter from the ISO date `from`, which starts one, up to but not including `until`."
+ [from until]
+ (let [until (LocalDate/parse until)]
+ (take-while #(.isBefore ^LocalDate % until)
+ (iterate #(.plusMonths ^LocalDate % 3) (LocalDate/parse from)))))
+
+(defn- quarterly-commits
+ "The first commit of `ref` and the last one before each quarter until the monthly sample starts: a sparse view of
+ the years before there were modules."
+ [ref]
+ (cons (first (git-lines ["rev-list" "--max-parents=0" "--first-parent" ref]))
+ (mapcat #(git-lines ["rev-list" "-1" "--first-parent" (str "--before=" % "T00:00:00Z") ref])
+ (quarter-starts first-quarter (:monthly-since history-sample)))))
+
+(defn- key-commits-with-parents
+ "Full shas of each of [[key-commits]] and its first parent."
+ []
+ (git-lines (into ["rev-parse"] (mapcat (juxt #(str % "^{commit}") #(str % "^"))) (keys key-commits))))
+
+(defn- cluster-anchors
+ "Cluster name -> anchor module, as `ref` has them."
+ [ref]
+ (->> (git-lines ["show" (str ref ":" cycle-names/clusters-file)])
+ (str/join "\n")
+ edn/read-string
+ cycle-names/validate-anchors))
+
+;;;; Rows
+
+(defn- short-sha [row]
+ (subs (:sha row) 0 11))
+
+(defn- in-order
+ "`rows` that have a position, one per commit, oldest first."
+ [position rows]
+ (->> rows
+ (filter (comp position :sha))
+ (into {} (map (juxt :sha identity)))
+ vals
+ (sort-by (comp position :sha))
+ vec))
+
+(defn- module-system?
+ "Whether `row` is from a commit that has the module system."
+ [row]
+ (not (false? (:module-system? row))))
+
+(defn- with-modules
+ "A function that is `f` of a row with the module system, and nil of a row from before it."
+ [f]
+ #(when (module-system? %) (f %)))
+
+(defn- whole-backend?
+ "Whether `cycle` is the one tangle that [[as-one-tangle]] reports before the module system."
+ [cycle]
+ (= #{'backend} (set (:modules cycle))))
+
+(defn- as-one-tangle
+ "The `row`, with one cycle holding every namespace when it is from before the module system."
+ [row]
+ ;; Before the module system the whole backend is one module, which a module graph cannot show as a cycle.
+ ;; Every later cycle split from that one tangle.
+ (if-not (module-system? row)
+ (merge row {:scc-largest-modules 1
+ :scc-largest-namespaces (:namespaces row)
+ :scc-cyclic-modules 1
+ :scc-cyclic-namespaces (:namespaces row)
+ :scc-components 1
+ :scc-static-largest-namespaces (:namespaces row)
+ :scc-dynamic-only-namespaces 0
+ :cycles [{:modules '[backend]
+ :namespaces (:namespaces row)
+ :edges 0
+ :density nil
+ :cycle-rank 0
+ :strong-bridges 0}]})
+ row))
+
+(defn- report-rows
+ "The `rows` [[in-order]], each through [[as-one-tangle]] and carrying its `:position`."
+ [position rows]
+ (mapv #(assoc (as-one-tangle %) :position (position (:sha %))) (in-order position rows)))
+
+(defn- cells
+ "The value of each of `columns` of `[name compute]` for `row`."
+ [columns row]
+ (mapv (fn [[_ compute]] (compute row)) columns))
+
+(defn- percent
+ "`part` of `whole` as a percentage with one decimal, in the same form under every locale."
+ [part whole]
+ (String/format Locale/ROOT "%.1f" (to-array [(if (zero? whole) 0.0 (* 100.0 (/ part whole)))])))
+
+;;;; history.csv
+
+(def ^:private history-columns
+ "Column of `history.csv` -> how to compute it from one of the [[report-rows]]."
+ [[:date #(subs (:committed-at %) 0 10)]
+ [:module_system module-system?]
+ [:committed_at :committed-at]
+ [:seq :position]
+ [:sha short-sha]
+ [:key_commit #(get key-commits (short-sha %) "")]
+ [:namespaces :namespaces]
+ [:modules :modules]
+ [:top_level_modules :top-level-modules]
+ [:nested_modules :nested-modules]
+ [:scc_largest_modules :scc-largest-modules]
+ [:scc_largest_namespaces :scc-largest-namespaces]
+ [:scc_largest_pct #(percent (:scc-largest-namespaces %) (:namespaces %))]
+ [:scc_cyclic_modules :scc-cyclic-modules]
+ [:scc_cyclic_namespaces :scc-cyclic-namespaces]
+ [:scc_cyclic_pct #(percent (:scc-cyclic-namespaces %) (:namespaces %))]
+ [:scc_components :scc-components]
+ [:scc_static_largest_namespaces :scc-static-largest-namespaces]
+ [:scc_dynamic_only_namespaces :scc-dynamic-only-namespaces]
+ [:module_edges :module-edges]
+ [:top_level_edges :top-level-edges]
+ [:upward_requires :upward-requires]
+ [:upward_module_edges :upward-module-edges]
+ [:upward_internal_requires :upward-internal-requires]
+ [:upward_cycle_requires :upward-cycle-requires]
+ ;; With one module there are no boundaries, so exposure means nothing before #42250.
+ [:exposed_namespaces (with-modules :exposed-namespaces)]
+ [:exposed_pct (with-modules #(percent (:exposed-namespaces %) (:namespaces %)))]
+ [:hidden_modules :hidden-modules]
+ [:hidden_namespaces :hidden-namespaces]
+ [:cross_top_level_required_namespaces :cross-top-level-required-namespaces]
+ [:api_any_modules :api-any-modules]
+ [:api_any_namespaces :api-any-namespaces]
+ [:uses_any_modules :uses-any-modules]
+ [:friend_grants :friend-grants]
+ [:cross_top_level_friend_grants :cross-top-level-friend-grants]
+ [:friend_exposed_namespaces :friend-exposed-namespaces]
+ [:friend_access_paths :friend-access-paths]
+ [:largest_ns_level_scc :largest-ns-level-scc]
+ [:mutual_dependency_pairs :mutual-dependency-pairs]
+ [:non_api_cross_module_requires :non-api-cross-module-requires]
+ [:friend_internal_requires :friend-internal-requires]
+ [:subject :subject]])
+
+(defn- table
+ "A header and one line per row, from `columns` of `[name compute]`."
+ [columns rows]
+ (cons (mapv (comp name first) columns)
+ (map #(cells columns %) rows)))
+
+(defn history-table
+ "Header and lines of `history.csv` for measured `rows`. `position` gives a commit's place on master by sha; a row
+ without one is left out."
+ [position rows]
+ (table history-columns (report-rows position rows)))
+
+;;;; structure.csv
+
+(defn- rounded-percent [part whole]
+ (if (zero? whole) 0.0 (/ (Math/round (* 1000.0 (/ part whole))) 10.0)))
+
+(def ^:private structure-id-columns
+ [[:committed_at :committed-at]
+ [:sha :sha]
+ [:pr #(some->> (re-find #"\(#(\d+)\)\s*$" (:subject %)) second)]
+ [:subject :subject]])
+
+(def ^:private structure-metric-columns
+ [[:modules :configured-modules]
+ [:top_level_modules :configured-top-level-modules]
+ [:namespaces :distinct-namespaces]
+ [:module_edges :module-edges]
+ [:cyclic_clusters :scc-components]
+ [:scc1_modules :scc1-modules]
+ [:scc1_namespaces :scc1-namespaces]
+ [:scc2_modules :scc2-modules]
+ [:scc2_namespaces :scc2-namespaces]
+ [:scc3_modules :scc3-modules]
+ [:scc3_namespaces :scc3-namespaces]
+ [:modules_in_cycles :scc-cyclic-modules]
+ [:namespaces_in_cycles :scc-cyclic-namespaces]
+ [:pct_namespaces_in_cycles #(rounded-percent (:scc-cyclic-namespaces %) (:distinct-namespaces %))]
+ [:largest_ns_level_scc :largest-ns-level-scc]
+ [:mutual_dependency_pairs :mutual-dependency-pairs]
+ [:api_any :api-any-modules]
+ [:uses_any :uses-any-modules]
+ [:friend_edges :friend-grants]
+ [:ns_prefixes :ns-prefixes]
+ [:model_imports_bypass :model-imports-bypass]
+ [:friend_internal_namespaces :friend-internal-namespaces]
+ [:friend_internal_requires :friend-internal-requires]
+ [:non_api_cross_module_requires :non-api-cross-module-requires]
+ [:total_api_namespaces :total-api-namespaces]
+ [:legacy_rest_modules :legacy-rest-modules]
+ [:upward_requires :upward-requires]
+ [:upward_cycle_requires :upward-cycle-requires]
+ [:upward_internal_requires :upward-internal-requires]
+ [:upward_module_edges :upward-module-edges]
+ [:exposed_namespaces :exposed-namespaces]
+ [:cross_top_level_required_namespaces :cross-top-level-required-namespaces]])
+
+(defn- ends-of-unchanged-runs
+ "The rows whose `values` differ from either neighbour's, plus the first and last: both ends of every run of
+ commits that changed nothing."
+ [values rows]
+ (let [rows (vec rows)
+ values (mapv values rows)
+ n (count rows)]
+ (for [i (range n)
+ :when (or (zero? i)
+ (= i (dec n))
+ (not= (values i) (values (dec i)))
+ (not= (values i) (values (inc i))))]
+ (rows i))))
+
+(defn structure-table
+ "Header and lines of `structure.csv` for measured `rows`: most commits move no metric, so each run of unchanged
+ commits keeps only its first and last."
+ [position rows]
+ (table (concat structure-id-columns structure-metric-columns)
+ (ends-of-unchanged-runs #(cells structure-metric-columns %) (in-order position rows))))
+
+;;;; cycles.csv
+
+(defn- cluster-names
+ "Cluster names of `anchors`, the one whose cycle has the most namespaces at `latest-row` first. A cluster that is
+ no longer a cycle there sorts last."
+ [anchors latest-row]
+ (let [size (fn [cluster]
+ (or (some (fn [cycle]
+ (when (some #{(anchors cluster)} (:modules cycle))
+ (:namespaces cycle)))
+ (:cycles latest-row))
+ 0))]
+ (sort-by (juxt (comp - size) str) (keys anchors))))
+
+(defn- cluster-name
+ "The first of `names` whose anchor `cycle` holds, else its [[gone-clusters]] entry, else `unnamed`.
+ A cycle that holds several anchors is those clusters before they split, and it takes the name of the largest.
+ The one tangle from before the module system takes the first of `names`."
+ [anchors names cycle]
+ (let [members (set (:modules cycle))]
+ (or (when (whole-backend? cycle) (str (first names)))
+ (some-> (first (filter #(contains? members (anchors %)) names)) str)
+ (some (fn [[cluster anchor]] (when (contains? members anchor) (str cluster))) gone-clusters)
+ "unnamed")))
+
+(defn- instant ^Instant [row]
+ (.toInstant (OffsetDateTime/parse (:committed-at row))))
+
+(defn- last-commit-of-each-instant
+ "The `rows` with only the last of any commits that share an instant."
+ [rows]
+ ;; A merge queue can land two commits in the same second, and a chart stacked over time would add them together.
+ (map last (partition-by instant rows)))
+
+(defn cycles-table
+ "Header and lines of `cycles.csv`: one line for each module cycle at each of `rows`, named by `anchors`, a map of
+ cluster name to anchor module. The `anchors` column counts the anchors a cycle holds, so a merge of today's
+ clusters shows."
+ [anchors position rows]
+ (let [rows (last-commit-of-each-instant (report-rows position rows))
+ names (cluster-names anchors (last rows))]
+ (cons ["committed_at" "seq" "sha" "cluster" "anchors" "modules" "namespaces" "edges" "density" "cycle_rank"
+ "strong_bridges" "members" "subject"]
+ (for [row rows
+ cycle (:cycles row)
+ :let [members (set (:modules cycle))]]
+ [(:committed-at row)
+ (:position row)
+ (short-sha row)
+ (cluster-name anchors names cycle)
+ (if (whole-backend? cycle)
+ (count anchors)
+ (count (filter (comp members anchors) names)))
+ (count members)
+ (:namespaces cycle)
+ (:edges cycle)
+ (:density cycle)
+ (:cycle-rank cycle)
+ (:strong-bridges cycle)
+ (str/join " " (:modules cycle))
+ (:subject row)]))))
+
+;;;; history.json
+
+(defn- cycle-holding
+ "The cycle at `row` that holds `module`. Before the module system that is the one tangle, whatever the module."
+ [row module]
+ (first (filter #(or (whole-backend? %) (some #{module} (:modules %))) (:cycles row))))
+
+(def ^:private commit-keys
+ "Key of a commit in `history.json` -> how to compute it from one of the [[report-rows]] that carries its
+ `:largest-cycle` and `:lanes`. Page templates kept outside this repository read these keys by name."
+ [;; When it was committed, with its UTC offset.
+ [:at :committed-at]
+ ;; The day it was committed.
+ [:date #(subs (:committed-at %) 0 10)]
+ ;; Short sha.
+ [:sha short-sha]
+ ;; Label of one of the [[key-commits]], else an empty string.
+ [:key #(get key-commits (short-sha %) "")]
+ ;; First line of the commit message.
+ [:subject :subject]
+ ;; Backend namespaces.
+ [:ns :namespaces]
+ ;; Namespaces in the largest cycle.
+ [:scc :scc-largest-namespaces]
+ ;; Modules in the largest cycle.
+ [:sccm :scc-largest-modules]
+ ;; Share of the namespaces that the largest cycle holds, as a percentage.
+ [:pct #(parse-double (percent (:scc-largest-namespaces %) (:namespaces %)))]
+ ;; Namespaces in any cycle.
+ [:cyc :scc-cyclic-namespaces]
+ ;; Namespaces that only dynamic requires keep in the largest cycle.
+ [:dyn :scc-dynamic-only-namespaces]
+ ;; Strong bridges of the largest cycle.
+ [:bridges (comp :strong-bridges :largest-cycle)]
+ ;; Cycle rank of the largest cycle.
+ [:rank (comp :cycle-rank :largest-cycle)]
+ ;; Mean share of the backend a change retests, selecting tests by module.
+ [:rtm :retest-module-mean]
+ ;; The same, selecting tests by namespace.
+ [:rtn :retest-namespace-mean]
+ ;; Leaf modules, which nothing else requires. Before modules existed the one module is not a leaf worth counting.
+ [:lfm (with-modules :leaf-modules)]
+ ;; Namespaces in leaf modules.
+ [:lfmn (with-modules :leaf-module-namespaces)]
+ ;; Leaf namespaces, which nothing else requires.
+ [:lfn :leaf-namespaces]
+ ;; Requires from a module into one of its ancestors.
+ [:up :upward-requires]
+ ;; Those of them that close a cycle.
+ [:upcyc :upward-cycle-requires]
+ ;; Those of them that reach past the ancestor's API.
+ [:upint :upward-internal-requires]
+ ;; Namespaces another top-level module may require. With one module there are no boundaries to expose them.
+ [:exp (with-modules :exposed-namespaces)]
+ ;; Non-API namespaces that a `:friends` grant opens to another top-level module.
+ [:fx :friend-exposed-namespaces]
+ ;; Non-API namespaces that a friend really requires.
+ [:fxu :friend-internal-namespaces]
+ ;; Namespaces required from another top-level module.
+ [:xreq :cross-top-level-required-namespaces]
+ ;; Top-level modules.
+ [:top :top-level-modules]
+ ;; Nested modules.
+ [:nested :nested-modules]
+ ;; Namespaces in the largest cycle of the namespace graph.
+ [:nslvl :largest-ns-level-scc]
+ ;; Pairs of modules that require each other, leaving out a module and its ancestor.
+ [:mutual :mutual-dependency-pairs]
+ ;; Requires of another module's non-API namespace.
+ [:nonapi :non-api-cross-module-requires]
+ ;; Cluster name -> the `[namespaces label]` of the cycle that holds its anchor, `[0 nil]` when there is none.
+ [:lanes :lanes]])
+
+(defn page-data
+ "What a page needs to chart measured `rows`, as a map ready for JSON:
+
+ - `:clusters`: every cluster name, today's largest first, then those that are gone
+ - `:commits`: one map per commit, under the short keys that [[commit-keys]] lists
+ - `:today`: the cycles at the latest commit, largest first
+ - `:retest`: at the latest commit, the namespaces in each tenth of retest share, selecting tests by `:module`
+ and by `:namespace`"
+ [anchors position rows]
+ (let [rows (report-rows position rows)
+ names (cluster-names anchors (last rows))
+ named (concat (map (juxt str anchors) names)
+ (map (fn [[cluster anchor]] [(str cluster) anchor]) gone-clusters))
+ lanes (fn [row]
+ (into {}
+ (for [[cluster anchor] named
+ :let [cycle (cycle-holding row anchor)]]
+ [cluster (if cycle [(:namespaces cycle) (cluster-name anchors names cycle)] [0 nil])])))
+ ;; The cycle with the most namespaces, which the `:scc-largest-*` counts describe.
+ largest #(when-let [cycles (seq (:cycles %))] (apply max-key :namespaces cycles))]
+ {:clusters (mapv first named)
+ :commits (for [row rows]
+ (zipmap (map first commit-keys)
+ (cells commit-keys (assoc row :largest-cycle (largest row), :lanes (lanes row)))))
+ :retest (:retest-histogram (last rows))
+ :today (for [cycle (sort-by (comp - :namespaces) (:cycles (last rows)))]
+ {:cluster (cluster-name anchors names cycle)
+ :modules (count (:modules cycle))
+ :namespaces (:namespaces cycle)
+ :edges (:edges cycle)
+ :density (:density cycle)
+ :rank (:cycle-rank cycle)
+ :bridges (:strong-bridges cycle)
+ :members (str/join " " (sort (map str (:modules cycle))))})}))
+
+;;;; Files
+
+(defn- write-atomically!
+ "Write `file` with `write`, a function of a writer, through a temporary file, so a reader never sees half a file."
+ [^File file write]
+ (io/make-parents file)
+ (let [tmp (File/createTempFile (.getName file) ".tmp" (.getParentFile file))]
+ (with-open [writer (io/writer tmp)]
+ (write writer))
+ (Files/move (.toPath tmp) (.toPath file)
+ (into-array CopyOption [StandardCopyOption/ATOMIC_MOVE StandardCopyOption/REPLACE_EXISTING]))))
+
+(defn- write-csv!
+ "Write `lines` to `file` and return how many follow the header. Nil is an empty cell."
+ [file lines]
+ (write-atomically! file #(csv/write-csv % (map (fn [line] (map (fn [cell] (if (nil? cell) "" (str cell))) line))
+ lines)))
+ (dec (count lines)))
+
+;;;; upward-pairs.csv and cuts.csv
+
+(def ^:private cut-require-limit
+ "A cut that severs more requires than this is not offered."
+ 30)
+
+(defn tip-tables
+ "The two tables about one commit's `deps` and `modules`, as maps ready for JSON:
+
+ - `:pairs`: [[module-cuts/upward-pairs]], with names as strings
+ - `:cuts`: [[module-cuts/cut-candidates]], each as `{:base, :of, :requires, :after, :pct, :freed, :edges,
+ :severed}`, where `:pct` is `:after` as a share of all namespaces, `:edges` has names as strings and
+ `:severed` lists each require as `from-namespace -> to-namespace`"
+ [deps modules]
+ {:pairs (for [pair (module-cuts/upward-pairs deps modules)]
+ (-> pair (update :module str) (update :ancestor str)))
+ :cuts (for [{:keys [base-modules cycle-modules requires edges largest-after freed]}
+ (module-cuts/cut-candidates deps cut-require-limit)]
+ {:base base-modules
+ :of cycle-modules
+ :requires (count requires)
+ :after largest-after
+ :pct (rounded-percent largest-after (count deps))
+ :freed freed
+ :edges (mapv (fn [[from to n]] [(str from) (str to) n]) edges)
+ :severed (mapv (fn [[from to]] (str from " -> " to)) requires)})})
+
+(defn- pairs-table [sha pairs]
+ (cons ["sha" "module" "ancestor" "requires" "internal"]
+ (for [{:keys [module ancestor requires internal]} pairs]
+ [sha module ancestor requires internal])))
+
+(defn- cuts-table [sha cuts]
+ (cons ["sha" "base_modules" "cycle_modules" "requires" "largest_cycle_after" "share_after_pct" "namespaces_freed"
+ "severed"]
+ (for [{:keys [base of requires after pct freed severed]} cuts]
+ [sha base of requires after pct freed (str/join "; " severed)])))
+
+(defn update!
+ "Measure the sample on `:ref` (default `origin/master`, as last fetched) and write `history.csv`,
+ `structure.csv`, `cycles.csv`, `upward-pairs.csv`, `cuts.csv` and `history.json` into `:dir`.
+ Commits measured before come from the cache.
+
+ clojure -X:dev dev.module-history-report/update! :dir '\"target/module-history\"'"
+ [{:keys [dir ref], :or {ref default-ref}}]
+ (when (str/blank? (str dir))
+ (throw (ex-info "update! needs :dir, the directory to write the files into" {})))
+ ;; Before any step that reads the history, so a shallow clone fails here and not on a missing parent.
+ (module-history/require-full-history!)
+ (let [dir (io/file (str dir))
+ ;; Every step reads this one commit, so a fetch during a long run cannot mix two tips in the files.
+ tip (first (git-lines ["rev-parse" (str ref "^{commit}")]))
+ history (module-history/backfill! (assoc history-sample
+ :ref tip
+ :extra (concat (quarterly-commits tip) (key-commits-with-parents))))
+ structure (module-history/backfill! (assoc structure-sample :ref tip))
+ position (positions tip)
+ anchors (cluster-anchors tip)
+ sha (subs tip 0 11)
+ tables (let [{:keys [deps modules]} (module-history/snapshot tip)] (tip-tables deps modules))
+ page (merge (page-data anchors position history) tables)]
+ (doseq [[file-name lines] [["history.csv" (history-table position history)]
+ ["structure.csv" (structure-table position structure)]
+ ["cycles.csv" (cycles-table anchors position (concat history structure))]
+ ["upward-pairs.csv" (pairs-table sha (:pairs tables))]
+ ["cuts.csv" (cuts-table sha (:cuts tables))]]]
+ (println (format "%-16s %5d rows" file-name (write-csv! (io/file dir file-name) lines))))
+ (write-atomically! (io/file dir "history.json") #(json/encode-to page % {}))
+ (println (format "%-16s %5d commits" "history.json" (count (:commits page))))
+ (println "Written to" (.getCanonicalPath dir))))
diff --git a/dev/test/metabase/core/module_cuts_test.clj b/dev/test/metabase/core/module_cuts_test.clj
new file mode 100644
index 00000000000..d4d13427fb1
--- /dev/null
+++ b/dev/test/metabase/core/module_cuts_test.clj
@@ -0,0 +1,105 @@
+(ns metabase.core.module-cuts-test
+ (:require
+ [clojure.test :refer :all]
+ [dev.module-cuts :as module-cuts]))
+
+(defn- row
+ "A dependency row for `namespace` in `module` that requires each `[namespace module]` of `requires`."
+ [namespace module requires]
+ {:namespace namespace
+ :module module
+ :deps (for [[required-namespace required-module] requires]
+ {:namespace required-namespace, :module required-module})})
+
+(deftest ^:parallel upward-pairs-test
+ ;; x requires its children x.y and x.z, so their requires back into x close a cycle. x.w requires x, and
+ ;; nothing requires x.w.
+ (let [modules '{x {}, x.y {}, x.z {}, x.w {}}
+ deps [(row 'metabase.x.core 'x '[[metabase.x.y.core x.y] [metabase.x.z.core x.z]])
+ (row 'metabase.x.impl 'x [])
+ (row 'metabase.x.y.core 'x.y '[[metabase.x.core x] [metabase.x.impl x]])
+ (row 'metabase.x.y.other 'x.y '[[metabase.x.impl x]])
+ (row 'metabase.x.z.core 'x.z '[[metabase.x.core x]])
+ (row 'metabase.x.w.core 'x.w '[[metabase.x.impl x]])]]
+ (testing "pairs that close a cycle are counted, with the requires that reach past the ancestor's API"
+ (is (= '[{:module x.y, :ancestor x, :requires 3, :internal 2}
+ {:module x.z, :ancestor x, :requires 1, :internal 0}]
+ (module-cuts/upward-pairs deps modules))))))
+
+(deftest ^:parallel cut-candidates-test
+ ;; A cycle of base, mid and top, three namespaces each. One require goes up from base into mid, and two go up
+ ;; from mid into top. An unrelated module requires base, and base.a requires inside its own module.
+ (let [deps [(row 'metabase.base.core 'base '[[metabase.mid.core mid]])
+ (row 'metabase.base.a 'base '[[metabase.base.core base]])
+ (row 'metabase.base.b 'base [])
+ (row 'metabase.mid.core 'mid '[[metabase.base.core base] [metabase.top.core top]])
+ (row 'metabase.mid.a 'mid '[[metabase.top.core top]])
+ (row 'metabase.mid.b 'mid [])
+ (row 'metabase.top.core 'top '[[metabase.mid.core mid] [metabase.base.core base]])
+ (row 'metabase.top.a 'top [])
+ (row 'metabase.top.b 'top [])
+ (row 'metabase.other.core 'other '[[metabase.base.core base]])]]
+ (testing "a bottom layer is offered when cutting it loose frees more than every cheaper cut, cheapest first"
+ ;; The order is base, top, mid. One require frees base; two leave no cycle at all.
+ (is (= '[{:base-modules 1
+ :cycle-modules 3
+ :requires [[metabase.base.core metabase.mid.core]]
+ :edges [[base mid 1]]
+ :largest-after 6
+ :freed 3}
+ {:base-modules 2
+ :cycle-modules 3
+ :requires [[metabase.base.core metabase.mid.core]
+ [metabase.top.core metabase.mid.core]]
+ :edges [[base mid 1] [top mid 1]]
+ :largest-after 0
+ :freed 9}]
+ (module-cuts/cut-candidates deps 30))))
+ (testing "the cycle with the most namespaces is the one cut, not the one with the most modules"
+ (is (= [{:cycle-modules 2, :edges '[[q p 1]], :largest-after 0, :freed 10}]
+ (map #(select-keys % [:cycle-modules :edges :largest-after :freed])
+ (module-cuts/cut-candidates (into deps
+ (concat [(row 'metabase.p.core 'p '[[metabase.q.core q]])
+ (row 'metabase.q.core 'q '[[metabase.p.core p]])]
+ (for [n (range 8)]
+ (row (symbol (str "metabase.p.n" n)) 'p []))))
+ 30)))))
+ (testing "another cycle in the graph does not count as what is left of this one"
+ (is (= [[1 6] [2 0]]
+ (map (juxt :base-modules :largest-after)
+ (module-cuts/cut-candidates (concat deps
+ [(row 'metabase.p.core 'p '[[metabase.q.core q]])
+ (row 'metabase.p.a 'p [])
+ (row 'metabase.q.core 'q '[[metabase.p.core p]])
+ (row 'metabase.q.a 'q [])])
+ 30)))))
+ (testing "a layer that costs as much as another and frees less is dropped, and the cheapest comes first"
+ ;; With far on top the order is top, mid, base, far. The bottom two cost three requires, as the bottom one
+ ;; does, and free less; the bottom three cost one. Requires are listed by module edge, then by namespace.
+ (is (= '[{:base-modules 3
+ :requires [[metabase.top.c metabase.far.core]]
+ :freed 1}
+ {:base-modules 1
+ :requires [[metabase.top.core metabase.base.core]
+ [metabase.top.c metabase.far.core]
+ [metabase.top.core metabase.mid.core]]
+ :freed 5}]
+ (map #(select-keys % [:base-modules :requires :freed])
+ (module-cuts/cut-candidates
+ (concat deps
+ [(row 'metabase.far.core 'far '[[metabase.top.core top]
+ [metabase.top.a top]
+ [metabase.mid.core mid]])
+ (row 'metabase.top.c 'top '[[metabase.far.core far]])])
+ 30)))))
+ (testing "a require inside one module is not an edge"
+ (is (not-any? (fn [[from to]] (= from to))
+ (keys (#'module-cuts/requires-by-edge deps)))))
+ (testing "a candidate needing more requires than the limit is left out"
+ (is (= [1]
+ (map :base-modules (module-cuts/cut-candidates deps 1)))))
+ (testing "without a cycle there is nothing to cut"
+ (is (= []
+ (module-cuts/cut-candidates [(row 'metabase.base.core 'base [])
+ (row 'metabase.top.core 'top '[[metabase.base.core base]])]
+ 30))))))
diff --git a/dev/test/metabase/core/module_history_report_test.clj b/dev/test/metabase/core/module_history_report_test.clj
new file mode 100644
index 00000000000..dc3fb944b8d
--- /dev/null
+++ b/dev/test/metabase/core/module_history_report_test.clj
@@ -0,0 +1,328 @@
+(ns metabase.core.module-history-report-test
+ (:require
+ [clojure.test :refer :all]
+ [dev.module-history-report :as report]
+ [metabase.util.json :as json]))
+
+(defn- lines->maps
+ "Each line after the header as a map of column name to value."
+ [[header & lines]]
+ (mapv #(zipmap (map keyword header) %) lines))
+
+(def ^:private position
+ "Places on master for the commits these tests use. `off-master` has none."
+ {"aaaaaaaaaaaa1" 10
+ "bbbbbbbbbbbb2" 11
+ "cccccccccccc3" 12
+ "dddddddddddd4" 13})
+
+(deftest ^:parallel quarter-starts-test
+ (testing "a quarter that starts on the end date is left out"
+ (is (= ["2015-04-01" "2015-07-01" "2015-10-01" "2016-01-01"]
+ (map str (#'report/quarter-starts "2015-04-01" "2016-04-01")))))
+ (testing "a quarter that starts before the end date is kept"
+ (is (= ["2015-04-01" "2015-07-01"]
+ (map str (#'report/quarter-starts "2015-04-01" "2015-07-02"))))))
+
+(deftest ^:parallel history-table-test
+ (let [rows [{:sha "bbbbbbbbbbbb2"
+ :committed-at "2026-01-02T10:00:00Z"
+ :subject "Carve \"x\", then y (#2)"
+ :module-system? true
+ :namespaces 200
+ :scc-largest-namespaces 50
+ :scc-cyclic-namespaces 75
+ :exposed-namespaces 30}
+ ;; Later on the clock than the commit above, but earlier on master.
+ {:sha "aaaaaaaaaaaa1"
+ :committed-at "2026-01-02T23:00:00+13:00"
+ :subject "Before the module system"
+ :module-system? false
+ :namespaces 120}
+ {:sha "off-master"
+ :committed-at "2026-01-03T00:00:00Z"
+ :subject "On a branch"
+ :namespaces 1}]]
+ (testing "rows follow master, not the clock, and a commit that is not on master is left out"
+ (is (=? [;; Before the module system the backend is one tangle of every namespace, with no exposure to report.
+ {:seq 10
+ :sha "aaaaaaaaaaa"
+ :date "2026-01-02"
+ :subject "Before the module system"
+ :module_system false
+ :scc_largest_modules 1
+ :scc_largest_namespaces 120
+ :scc_largest_pct "100.0"
+ :scc_cyclic_pct "100.0"
+ :scc_components 1
+ :exposed_namespaces nil
+ :exposed_pct nil}
+ ;; Percentages have one decimal.
+ {:seq 11
+ :sha "bbbbbbbbbbb"
+ :date "2026-01-02"
+ :subject "Carve \"x\", then y (#2)"
+ :module_system true
+ :scc_largest_pct "25.0"
+ :scc_cyclic_pct "37.5"
+ :exposed_pct "15.0"}]
+ (lines->maps (report/history-table position rows)))))))
+
+(deftest ^:parallel key-commit-label-test
+ (let [[sha label] (first report/key-commits)]
+ (is (=? [{:key_commit label} {:key_commit ""}]
+ (lines->maps (report/history-table {(str sha "0000") 1, "bbbbbbbbbbbb2" 2}
+ (for [commit [(str sha "0000") "bbbbbbbbbbbb2"]]
+ {:sha commit
+ :committed-at "2026-01-01T00:00:00Z"
+ :namespaces 10
+ :scc-largest-namespaces 0
+ :scc-cyclic-namespaces 0
+ :exposed-namespaces 0})))))))
+
+(deftest ^:parallel structure-table-test
+ (let [row (fn [sha subject namespaces]
+ {:sha sha
+ :committed-at "2026-09-01T00:00:00Z"
+ :subject subject
+ :distinct-namespaces namespaces
+ :scc-cyclic-namespaces 0})
+ rows [(row "aaaaaaaaaaaa1" "Start a run (#1)" 100)
+ (row "bbbbbbbbbbbb2" "Change nothing (#2)" 100)
+ (row "cccccccccccc3" "End the run" 100)
+ (row "dddddddddddd4" "Add a namespace (#4)" 101)]]
+ (testing "a run of commits that move no metric keeps its two ends, and the PR number comes from the subject"
+ (is (=? [{:sha "aaaaaaaaaaaa1", :pr "1", :namespaces 100}
+ {:sha "cccccccccccc3", :pr nil, :namespaces 100}
+ {:sha "dddddddddddd4", :pr "4", :namespaces 101}]
+ (lines->maps (report/structure-table position (reverse rows))))))))
+
+(deftest ^:parallel cycles-table-test
+ (let [anchors '{galactic-center query-processor
+ deep-thought metabot
+ tardis lib}
+ cycle (fn [modules namespaces]
+ {:modules modules
+ :namespaces namespaces
+ :edges 9
+ :density 0.5
+ :cycle-rank 2
+ :strong-bridges 1})
+ rows [{:sha "aaaaaaaaaaaa1"
+ :committed-at "2026-01-01T00:00:00Z"
+ :subject "No modules yet"
+ :module-system? false
+ :namespaces 700}
+ {:sha "bbbbbbbbbbbb2"
+ :committed-at "2026-01-02T00:00:00Z"
+ :subject "One big cycle"
+ :cycles [(cycle '[lib metabot query-processor] 600)
+ (cycle '[segments xrays] 20)
+ (cycle '[a b] 5)]}
+ ;; The same second as the commit after it, written with another offset.
+ {:sha "cccccccccccc3"
+ :committed-at "2026-01-03T02:00:00+02:00"
+ :subject "Dropped"
+ :cycles [(cycle '[lib metabot query-processor] 600)]}
+ {:sha "dddddddddddd4"
+ :committed-at "2026-01-03T00:00:00Z"
+ :subject "Split"
+ ;; metabot's cycle is the largest at the latest commit, so its name goes first in a merge.
+ :cycles [(cycle '[metabot slackbot] 300)
+ (cycle '[lib query-processor] 200)]}]]
+ (is (=? [{:seq 10
+ :cluster "deep-thought"
+ :anchors 3
+ :modules 1
+ :namespaces 700
+ :members "backend"}
+ {:seq 11
+ :cluster "deep-thought"
+ :anchors 3
+ :modules 3
+ :namespaces 600
+ :members "lib metabot query-processor"
+ :density 0.5
+ :cycle_rank 2
+ :strong_bridges 1
+ :edges 9}
+ {:seq 11
+ :cluster "magellanic-cloud"
+ :anchors 0
+ :members "segments xrays"}
+ {:seq 11
+ :cluster "unnamed"
+ :anchors 0
+ :members "a b"}
+ {:seq 13
+ :cluster "deep-thought"
+ :anchors 1
+ :sha "ddddddddddd"
+ :subject "Split"}
+ ;; tardis ties with galactic-center on this cycle's size, and the names then sort by letter.
+ {:seq 13
+ :cluster "galactic-center"
+ :anchors 2
+ :members "lib query-processor"}]
+ (lines->maps (report/cycles-table anchors position rows))))))
+
+(def ^:private page-anchors
+ '{galactic-center query-processor
+ tardis lib})
+
+(def ^:private page-rows
+ "A commit from before the module system and one after it, for [[report/page-data]]."
+ (let [counts {:namespaces 100
+ :scc-largest-namespaces 60
+ :exposed-namespaces 40
+ :retest-module-mean 70.0
+ :retest-namespace-mean 20.0
+ :retest-histogram {:module [1 0 0 0 0 0 0 0 0 9], :namespace [9 1]}
+ :leaf-modules 3
+ :leaf-module-namespaces 12
+ :leaf-namespaces 30
+ :friend-internal-namespaces 4}]
+ [(merge counts {:sha "aaaaaaaaaaaa1"
+ :committed-at "2026-01-01T00:00:00Z"
+ :subject "No modules yet"
+ :module-system? false})
+ (merge counts {:sha "bbbbbbbbbbbb2"
+ :committed-at "2026-01-02T00:00:00Z"
+ :subject "Split (#2)"
+ :cycles [{:modules '[lib lib.schema]
+ :namespaces 20
+ :density 0.5
+ :cycle-rank 1
+ :strong-bridges 2
+ :edges 2}
+ {:modules '[query-processor b a]
+ :namespaces 60
+ :density 0.4
+ :cycle-rank 3
+ :strong-bridges 0
+ :edges 5}]})]))
+
+(deftest ^:parallel page-data-test
+ (is (=? {:clusters ["galactic-center" "tardis" "magellanic-cloud" "sub-etha"]
+ :retest {:module [1 0 0 0 0 0 0 0 0 9], :namespace [9 1]}
+ :commits [{:sha "aaaaaaaaaaa"
+ :pct 100.0
+ :lfm nil
+ :lfn 30
+ :exp nil
+ :bridges 0
+ :rank 0
+ ;; Every cluster was inside the one tangle.
+ :lanes {"galactic-center" [100 "galactic-center"]
+ "tardis" [100 "galactic-center"]
+ "magellanic-cloud" [100 "galactic-center"]}}
+ {:sha "bbbbbbbbbbb"
+ :date "2026-01-02"
+ :subject "Split (#2)"
+ :ns 100
+ :scc 60
+ :pct 60.0
+ :rtm 70.0
+ :rtn 20.0
+ :fxu 4
+ :lfm 3
+ :lfmn 12
+ :lfn 30
+ :exp 40
+ ;; Of the cycle with the most namespaces.
+ :bridges 0
+ :rank 3
+ :lanes {"galactic-center" [60 "galactic-center"]
+ "tardis" [20 "tardis"]
+ "magellanic-cloud" [0 nil]
+ "sub-etha" [0 nil]}}]
+ :today [{:cluster "galactic-center"
+ :modules 3
+ :namespaces 60
+ :members "a b query-processor"
+ :rank 3
+ :bridges 0}
+ {:cluster "tardis"
+ :modules 2
+ :namespaces 20
+ :members "lib lib.schema"}]}
+ (report/page-data page-anchors position page-rows))))
+
+(deftest ^:parallel page-data-json-test
+ (testing "the page data has exactly the keys page templates read, before the tip tables are added"
+ (let [page (json/decode (json/encode (report/page-data page-anchors position page-rows)))]
+ (is (= {:page #{"clusters" "commits" "retest" "today"}
+ :commit #{"at"
+ "bridges"
+ "cyc"
+ "date"
+ "dyn"
+ "exp"
+ "fx"
+ "fxu"
+ "key"
+ "lanes"
+ "lfm"
+ "lfmn"
+ "lfn"
+ "mutual"
+ "nested"
+ "nonapi"
+ "ns"
+ "nslvl"
+ "pct"
+ "rank"
+ "rtm"
+ "rtn"
+ "scc"
+ "sccm"
+ "sha"
+ "subject"
+ "top"
+ "up"
+ "upcyc"
+ "upint"
+ "xreq"}}
+ {:page (set (keys page))
+ :commit (set (keys (first (get page "commits"))))})))))
+
+(deftest ^:parallel tip-tables-test
+ ;; x requires its child x.y, which requires x's API and one internal namespace back. Cutting the one require
+ ;; from x into x.y leaves no cycle.
+ (let [dep (fn [namespace module] {:namespace namespace, :module module})
+ deps [{:namespace 'metabase.x.core, :module 'x, :deps [(dep 'metabase.x.y.core 'x.y)]}
+ {:namespace 'metabase.x.impl, :module 'x, :deps []}
+ {:namespace 'metabase.x.y.core
+ :module 'x.y
+ :deps [(dep 'metabase.x.core 'x) (dep 'metabase.x.impl 'x)]}
+ {:namespace 'metabase.z.core, :module 'z, :deps []}]]
+ (is (= {:pairs [{:module "x.y", :ancestor "x", :requires 2, :internal 1}]
+ :cuts [{:base 1
+ :of 2
+ :requires 1
+ :after 0
+ :pct 0.0
+ :freed 3
+ :edges [["x" "x.y" 1]]
+ :severed ["metabase.x.core -> metabase.x.y.core"]}]}
+ (report/tip-tables deps '{x {}, x.y {}, z {}})))))
+
+(deftest ^:parallel tip-table-columns-test
+ (testing "the two tip tables keep the columns their uploads were created with"
+ (is (= [{:sha "abc", :module "x.y", :ancestor "x", :requires 2, :internal 1}]
+ (lines->maps (#'report/pairs-table "abc" [{:module "x.y", :ancestor "x", :requires 2, :internal 1}]))))
+ (is (= [{:sha "abc"
+ :base_modules 1
+ :cycle_modules 2
+ :requires 2
+ :largest_cycle_after 0
+ :share_after_pct 0.0
+ :namespaces_freed 3
+ :severed "a.core -> b.core; a.impl -> b.core"}]
+ (lines->maps (#'report/cuts-table "abc" [{:base 1
+ :of 2
+ :requires 2
+ :after 0
+ :pct 0.0
+ :freed 3
+ :severed ["a.core -> b.core" "a.impl -> b.core"]}]))))))
diff --git a/dev/test/metabase/core/module_history_test.clj b/dev/test/metabase/core/module_history_test.clj
index 47fa95bc064..8930d28d53a 100644
--- a/dev/test/metabase/core/module_history_test.clj
+++ b/dev/test/metabase/core/module_history_test.clj
@@ -6,7 +6,8 @@
[dev.deps-graph :as deps-graph]
[dev.module-history :as module-history]
[dev.module-source :as module-source]
- [hooks.common.modules]))
+ [hooks.common.modules]
+ [metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -291,8 +292,8 @@
(deftest commits-to-measure-test
(testing "an explicit revision is measured once when the sample also picks it"
- (let [head (str/trim (#'module-history/git! ["rev-parse" "HEAD"]))
- ;; A date after every commit samples only the tip of `ref`.
- tomorrow (str (.plusDays (java.time.LocalDate/now java.time.ZoneOffset/UTC) 1))]
- (is (= [head]
- (#'module-history/commits-to-measure {:commits ["HEAD"], :ref "HEAD", :since tomorrow}))))))
+ ;; Stubbed, because the sample walks git history, which a shallow CI checkout does not have.
+ (mt/with-dynamic-fn-redefs [module-history/resolve-commit (fn [commit] (get {"HEAD" "abc"} commit commit))
+ module-history/sample-commits (constantly ["abc" "def"])]
+ (is (= ["abc" "def"]
+ (#'module-history/commits-to-measure {:commits ["HEAD"], :since "2026-01-01"}))))))
diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.clj
index bec96f539b4..fd8e7713907 100644
--- a/mage/src/mage/jar_download.clj
+++ b/mage/src/mage/jar_download.clj
@@ -3,8 +3,6 @@
[babashka.curl :as curl]
[babashka.fs :as fs]
[babashka.process :as p]
- ;; mage runs under babashka, which bundles cheshire; metabase.util.json isn't on its classpath
- ^{:clj-kondo/ignore [:discouraged-namespace]}
[cheshire.core :as json]
[clojure.edn :as edn]
[clojure.java.io :as io]
diff --git a/mage/src/mage/module_history.clj b/mage/src/mage/module_history.clj
new file mode 100644
index 00000000000..7645ccde5eb
--- /dev/null
+++ b/mage/src/mage/module_history.clj
@@ -0,0 +1,332 @@
+(ns mage.module-history
+ "Keeps the files that track how the backend module graph changed up to date, and publishes them.
+
+ The measuring and the files are `dev.module-history-report`'s work. Publishing is described by an EDN file kept
+ outside the repo, since it names one person's or one team's uploads, cards and pages:
+
+ {:pages [{:template \"explorer-template.html\", :file \"module-boundaries.html\"}]
+ :metabase {:profile \"stats\"
+ :uploads {\"history.csv\" 123, \"structure.csv\" 124, \"cycles.csv\" 125}
+ :cards [456 457]}
+ :shares {\"module-boundaries.html\" \"module-boundaries-a1b2c3\"}}
+
+ - `:pages` fills each `:template`, a path relative to the EDN file, from `history.json` and writes it to `:file`
+ in the output directory. See [[fill-template]] for the placeholders.
+ - `:metabase` replaces the rows of each upload table with a file from the output directory, using the `mb` CLI
+ and its `:profile`. A replace needs the same columns as the upload. Metabase caches a card's results by its
+ query, so each of the native `:cards` then gets a new comment at the end of its SQL.
+ - `:shares` overwrites each mb-shares slug with a file from the output directory, using the `share` CLI."
+ (:require
+ [babashka.fs :as fs]
+ [cheshire.core :as json]
+ [clojure.edn :as edn]
+ [clojure.string :as str]
+ [mage.color :as c]
+ [mage.shell :as shell]
+ [mage.util :as u])
+ (:import
+ (java.time Instant)
+ (java.time.temporal ChronoUnit)
+ (java.util.regex Pattern)))
+
+(set! *warn-on-reflection* true)
+
+;;;; Running commands
+
+(defn- sh!
+ "Run `args` and return its stdout lines, exiting when the command fails."
+ [opts args]
+ (let [{:keys [exit out err]} (apply shell/sh* opts args)]
+ (when-not (zero? exit)
+ (u/exit (c/red (str "Failed: " (str/join " " args)
+ (when (seq err) (str "\n" (str/join "\n" err)))))
+ exit))
+ out))
+
+(defn- change!
+ "Run `args`, which changes something outside this checkout. With `dry-run?`, print the command instead."
+ [dry-run? args]
+ (if dry-run?
+ (println (c/yellow "Would run:") (str/join " " args))
+ (sh! {} args)))
+
+;;;; Measuring
+
+(def ^:private measuring-timeout-ms
+ "Enough to measure the whole sample with an empty cache."
+ (* 3 60 60 1000))
+
+(defn- measure!
+ "Write the files into `dir`, for the history of `ref` when it is given and of `origin/master` otherwise."
+ [dir ref]
+ ;; Always a fresh JVM, never the dev REPL: results are cached under a hash of the measuring code on disk, and a
+ ;; REPL may have loaded an older version of it.
+ (sh! {:timeout-ms measuring-timeout-ms}
+ (cond-> ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" (pr-str (str dir))]
+ ref (conj ":ref" (pr-str ref)))))
+
+;;;; Publishing config
+
+(defn- unknown-keys
+ "A row for [[config-checks]]: the map at `path`, or each map in the sequence there, holds only `known` keys.
+ Any other value passes, and is left to the row that checks its shape."
+ [path known]
+ (let [unknown (fn [value]
+ (let [maps (cond (map? value) [value], (sequential? value) (filter map? value))]
+ (distinct (mapcat #(remove known (keys %)) maps))))]
+ [path
+ (comp empty? unknown)
+ #(str "has unknown keys: " (str/join ", " (unknown %)))]))
+
+(defn- map-from-strings-to
+ "A predicate for a map from strings to values that pass `valid-value?`."
+ [valid-value?]
+ (fn [m] (and (map? m) (every? string? (keys m)) (every? valid-value? (vals m)))))
+
+(defn- page? [page]
+ (and (map? page) (string? (:template page)) (string? (:file page))))
+
+(def ^:private config-checks
+ "What a publishing config must satisfy, as `[path valid? message]` in the order problems are reported.
+ A check is skipped when nothing is at `path`.
+ A `message` is a string, or a function of the value there."
+ [(unknown-keys [] #{:metabase :pages :shares})
+ (unknown-keys [:pages] #{:file :template})
+ [[:pages]
+ #(and (sequential? %) (every? page? %))
+ "must be a vector of {:template \"path\", :file \"name\"}"]
+ (unknown-keys [:metabase] #{:cards :profile :uploads})
+ [[:metabase]
+ #(string? (:profile %))
+ "needs :profile, the mb profile to use"]
+ [[:metabase :uploads]
+ (map-from-strings-to pos-int?)
+ "must map a file name to an upload table id"]
+ [[:metabase :cards]
+ #(and (sequential? %) (every? pos-int? %))
+ "must be a vector of card ids"]
+ [[:shares]
+ (map-from-strings-to string?)
+ "must map a file name to an mb-shares slug"]])
+
+(defn- config-problems
+ "Everything wrong with publishing `config`, as `{:path [:metabase :cards], :message \"must be ...\"}` maps.
+ The path is empty for the config itself. Empty when the config is usable."
+ [config]
+ (if-not (map? config)
+ [{:path [], :message "must hold one map"}]
+ (for [[path valid? message] config-checks
+ :let [value (get-in config path)]
+ :when (and (some? value) (not (valid? value)))]
+ {:path path
+ :message (if (fn? message) (message value) message)})))
+
+(defn- problem-line
+ [{:keys [path message]}]
+ (str (if (seq path) (str/join " " path) "the config") " " message))
+
+(defn- read-config
+ "The publishing config in `file`. Exits when the file is missing or malformed."
+ [file]
+ (when-not (fs/exists? file)
+ (u/exit (c/red (str "No publishing config at " file)) 1))
+ (let [config (try
+ (edn/read-string (slurp (str file)))
+ (catch Exception e
+ (u/exit (c/red (str file " is not readable EDN: " (ex-message e))) 1)))
+ problems (config-problems config)]
+ (when (seq problems)
+ (u/exit (c/red (str file " is not a publishing config:\n " (str/join "\n " (map problem-line problems)))) 1))
+ config))
+
+(defn- missing-tools
+ "The command-line tools that publishing `config` runs and that are not installed."
+ [{:keys [metabase shares]}]
+ (binding [u/*skip-warning* true]
+ (into [] (remove u/can-run?) (cond-> []
+ (or (seq (:uploads metabase)) (seq (:cards metabase))) (conj "mb")
+ (seq shares) (conj "share")))))
+
+;;;; Pages
+
+(defn- script-safe-json
+ "`value` as JSON that can sit inside a `<script>` element. Every `<` is escaped, since `</script>` ends the element
+ and `<!--` can make the parser skip the tag that would end it. So are the line and paragraph separators, which
+ older JavaScript parsers do not accept inside a string."
+ [value]
+ (str/replace (json/generate-string value) #"[<\u2028\u2029]" #(format "\\u%04x" (int (first %)))))
+
+(def ^:private placeholders
+ "Placeholder in a page template -> the key of `history.json` it shows, and how to write that key's value."
+ [["/*DATA*/" "commits" script-safe-json]
+ ["/*TODAY*/" "today" script-safe-json]
+ ["/*ANCHORS*/" "clusters" script-safe-json]
+ ["/*RETEST*/" "retest" script-safe-json]
+ ["/*PAIRS*/" "pairs" script-safe-json]
+ ["/*CUTS*/" "cuts" script-safe-json]
+ ["/*COUNT*/" "commits" count]])
+
+(defn fill-template
+ "The page `template` with its placeholders replaced from `page-data`, the contents of `history.json`:
+
+ - `/*DATA*/`: the commits, a JSON array
+ - `/*TODAY*/`: the cycles at the latest commit, a JSON array
+ - `/*ANCHORS*/`: the cluster names, a JSON array
+ - `/*RETEST*/`: the latest commit's namespaces per tenth of retest share, a JSON object
+ - `/*PAIRS*/`: the module pairs whose upward requires close a cycle at the latest commit, a JSON array
+ - `/*CUTS*/`: the cheapest ways to split the largest cycle at the latest commit, a JSON array
+ - `/*COUNT*/`: how many commits there are"
+ [template page-data]
+ ;; One pass over the template, so a placeholder inside a value already written, say in a commit subject, stays.
+ (let [by-placeholder (into {} (map (juxt first rest)) placeholders)]
+ (str/replace template
+ (re-pattern (str/join "|" (map (comp #(Pattern/quote %) first) placeholders)))
+ (fn [placeholder]
+ (let [[k write] (by-placeholder placeholder)]
+ (str (write (get page-data k))))))))
+
+(defn- missing-page-keys
+ "Keys that a placeholder in `template` shows and that `page-data` lacks."
+ [template page-data]
+ (distinct (for [[placeholder k] placeholders
+ :when (and (str/includes? template placeholder) (not (contains? page-data k)))]
+ k)))
+
+(defn- read-template
+ "The page template at `path`. Exits when there is none, or when it shows a key that `page-data` lacks."
+ [path page-data]
+ (when-not (fs/exists? path)
+ (u/exit (c/red (str "No page template at " path)) 1))
+ (let [template (slurp (str path))
+ missing (missing-page-keys template page-data)]
+ (when (seq missing)
+ (u/exit (c/red (str "history.json has no " (str/join ", " (map pr-str missing)) ", which " path
+ " shows. Run without --no-measure to write it again."))
+ 1))
+ template))
+
+(defn- write-pages!
+ "Fill each of `pages` and write it into `dir`. A template path is relative to `config-dir`."
+ [dir config-dir pages]
+ (when (seq pages)
+ (let [page-data (json/parse-string (slurp (str (fs/path dir "history.json"))))
+ ;; Every template is read and checked before any page is written.
+ templates (mapv #(read-template (fs/path config-dir (:template %)) page-data) pages)]
+ (doseq [[{:keys [file]} template] (map vector pages templates)]
+ (spit (str (fs/path dir file)) (fill-template template page-data))
+ (println (c/green "Wrote") (str (fs/path dir file)))))))
+
+;;;; Metabase
+
+(def ^:private refresh-comment-prefix "-- refreshed by mage module-history at ")
+
+(defn refreshed-sql
+ "Native `sql` ending in one comment that carries `stamp`, replacing the comment an earlier refresh left."
+ [sql stamp]
+ (let [lines (->> (str/split-lines sql)
+ (remove #(str/starts-with? % refresh-comment-prefix))
+ (str/join "\n"))]
+ (str (str/trimr lines) "\n" refresh-comment-prefix stamp)))
+
+(defn- mb
+ "An `mb` command line that asks `profile` for JSON without a size cap."
+ [profile args]
+ (concat ["mb"] args ["--profile" profile "--json" "--max-bytes" "0"]))
+
+(defn- native-query
+ "The dataset query of native card `id`. Exits when the card is not native, or when `profile` may not change it."
+ [profile id]
+ ;; String keys, so that names inside the query, such as template tags, are sent back as they came.
+ (let [card (json/parse-string (str/join "\n" (sh! {:quiet? true} (mb profile ["card" "get" (str id) "--full"]))))
+ query (get card "dataset_query")]
+ (when-not (string? (get-in query ["stages" 0 "native"]))
+ (u/exit (c/red (str "Card " id " is not a native query, so its results cannot be refreshed this way")) 1))
+ (when (false? (get card "can_write"))
+ (u/exit (c/red (str "Card " id " cannot be changed with the " profile " profile")) 1))
+ query))
+
+(defn- refresh-card!
+ "Change the SQL of native card `id` so Metabase stops serving the results it cached."
+ [dry-run? profile stamp id]
+ (if dry-run?
+ (println (c/yellow "Would refresh card") id)
+ ;; Fetch the query again: an edit made while the uploads ran must not be overwritten with an older copy.
+ (let [query (native-query profile id)
+ tmp (fs/create-temp-dir {:prefix "module-history"})
+ body (str (fs/path tmp "card.json"))]
+ (try
+ (spit body (json/generate-string
+ {"dataset_query" (update-in query ["stages" 0 "native"] refreshed-sql stamp)}))
+ (sh! {:quiet? true} (mb profile ["card" "update" (str id) "--file" body]))
+ (finally
+ (fs/delete-tree tmp)))
+ (println (c/green "Refreshed card") id))))
+
+(defn- publish-to-metabase!
+ "Replace the upload tables, then refresh the cards. Every card is fetched and checked first, a dry run's too, so a
+ card that cannot be refreshed stops the run before any table changes."
+ [dry-run? dir {:keys [profile uploads cards]}]
+ (run! #(native-query profile %) cards)
+ (let [stamp (str (.truncatedTo (Instant/now) ChronoUnit/SECONDS))]
+ (doseq [[file table-id] (sort uploads)]
+ (change! dry-run? (mb profile ["upload" "replace" (str table-id) "--file" (str (fs/path dir file))])))
+ (doseq [id cards]
+ (refresh-card! dry-run? profile stamp id))))
+
+;;;; Publishing
+
+(defn- missing-files
+ "Files the config publishes that are not in `dir` and that no page will write there."
+ [dir {:keys [pages metabase shares]}]
+ (->> (concat (keys (:uploads metabase)) (keys shares))
+ (remove (set (map :file pages)))
+ (remove #(fs/exists? (fs/path dir %)))
+ distinct))
+
+(defn- publish!
+ "Publish the files in `dir` as `config` says, with page templates relative to `config-dir`. With `dry-run?`, write
+ the pages but only print the commands that would change Metabase or a share."
+ [dry-run? dir config config-dir]
+ (let [missing (missing-files dir config)]
+ (when (seq missing)
+ (u/exit (c/red (str "Not in " dir ": " (str/join ", " missing))) 1))
+ (write-pages! dir config-dir (:pages config))
+ (when-let [metabase (:metabase config)]
+ (publish-to-metabase! dry-run? dir metabase))
+ (doseq [[file slug] (sort (:shares config))]
+ (change! dry-run? ["share" "-e" (str (fs/path dir file)) slug]))))
+
+;;;; Entry point
+
+(defn- remote-branch
+ "The branch on `origin` that `ref` tracks, or nil for a local branch or a commit, which no fetch changes."
+ [ref]
+ (second (re-matches #"(?:refs/remotes/)?origin/(.+)" ref)))
+
+(defn cli-update
+ "Fetch the ref to measure, measure its new commits, write the files, and publish them when asked."
+ [{:keys [options]}]
+ (let [{:keys [dir publish ref no-fetch no-measure dry-run]} options]
+ (when (str/blank? dir)
+ (u/exit (c/red "Pass --dir DIR, the directory to keep the files in.") 1))
+ (when (and dry-run (not publish))
+ (u/exit (c/red "--dry-run only applies to --publish.") 1))
+ (let [dir (fs/absolutize (fs/expand-home dir))
+ config-file (some-> publish fs/expand-home)
+ ;; Read the config first, so a mistake in it shows before the measuring, not after.
+ config (some-> config-file read-config)
+ timer (u/start-timer)]
+ ;; A dry run checks too, so that it predicts what a real run does.
+ (when-let [missing (seq (some-> config missing-tools))]
+ (u/exit (c/red (str "Not installed, but needed to publish: " (str/join ", " missing))) 1))
+ (when-not no-measure
+ (when-not no-fetch
+ (when-let [branch (remote-branch (or ref "origin/master"))]
+ (println (str "Fetching origin/" branch "..."))
+ ;; Name the destination: a clone that fetches only some branches would update only FETCH_HEAD.
+ (sh! {} ["git" "fetch" "--quiet" "origin" (str "+refs/heads/" branch ":refs/remotes/origin/" branch)])))
+ (measure! dir ref))
+ (when config
+ (publish! (boolean dry-run) dir config (fs/parent (fs/absolutize config-file))))
+ (printf "\nFinished in %.1fs\n" (/ (u/since-ms timer) 1000.0))
+ (flush))))
diff --git a/mage/src/mage/project_tests.clj b/mage/src/mage/project_tests.clj
index 10292f91a6e..812e6bbf850 100644
--- a/mage/src/mage/project_tests.clj
+++ b/mage/src/mage/project_tests.clj
@@ -9,7 +9,10 @@
(def ^:private module-check-namespaces
'[dev.modules-config-test
+ metabase.core.module-cuts-test
metabase.core.module-cycles-test
+ metabase.core.module-history-report-test
+ metabase.core.module-history-test
metabase.core.modules-test])
(def ^:private honey-sql-check-namespaces
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index 87e1804a0e8..ffc61201b38 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -23,6 +23,7 @@
[mage.kondo-ratchets-history-test]
[mage.merge-kondo-ratchets-test]
[mage.merge-yaml-migrations-test]
+ [mage.module-history-test]
[mage.modules-test]
[mage.new-migration-test]
[mage.openapi-diff-test]
diff --git a/mage/test/mage/module_history_test.clj b/mage/test/mage/module_history_test.clj
new file mode 100644
index 00000000000..cd60f1aa6b6
--- /dev/null
+++ b/mage/test/mage/module_history_test.clj
@@ -0,0 +1,258 @@
+(ns mage.module-history-test
+ (:require
+ [babashka.fs :as fs]
+ [clojure.string :as str]
+ [clojure.test :refer [are deftest is testing]]
+ [mage.module-history :as module-history]
+ [mage.shell :as shell]
+ [mage.util :as u]))
+
+;; Referenced by core_test.clj to ensure namespace is loaded
+(def keep-me :loaded)
+
+(set! *warn-on-reflection* true)
+
+(deftest refreshed-sql-test
+ (testing "the stamp goes in a comment after the query"
+ (is (= "select 1\n-- refreshed by mage module-history at 2026-10-10T00:00:00Z"
+ (module-history/refreshed-sql "select 1\n\n\n" "2026-10-10T00:00:00Z"))))
+ (testing "a second refresh replaces the first one's comment, and other comments stay"
+ (is (= "select 1\n-- by seq\norder by seq\n-- refreshed by mage module-history at 2026-10-11T00:00:00Z"
+ (-> "select 1\n-- by seq\norder by seq"
+ (module-history/refreshed-sql "2026-10-10T00:00:00Z")
+ (module-history/refreshed-sql "2026-10-11T00:00:00Z"))))))
+
+(deftest fill-template-test
+ (is (= (str "<p>4 commits</p><script>"
+ "const DATA = [{\"subject\":\"Close \\u003c/script> early\"},"
+ "{\"subject\":\"Hide \\u003c!--\\u003cscript> \\u2028\"},"
+ "{\"subject\":\"$1 \\\\ kept\"},"
+ "{\"subject\":\"Rename /*COUNT*/ and /*DATA*/\"}];"
+ "const TODAY = [{\"cluster\":\"tardis\"}];"
+ "const ANCHORS = [\"tardis\"];"
+ "const RETEST = {\"module\":[1,2]};"
+ "const PAIRS = [{\"module\":\"x.y\"}];"
+ "const CUTS = [];"
+ "</script>")
+ (module-history/fill-template
+ (str "<p>/*COUNT*/ commits</p><script>"
+ "const DATA = /*DATA*/;const TODAY = /*TODAY*/;"
+ "const ANCHORS = /*ANCHORS*/;const RETEST = /*RETEST*/;"
+ "const PAIRS = /*PAIRS*/;const CUTS = /*CUTS*/;</script>")
+ {"commits" [{"subject" "Close </script> early"}
+ {"subject" "Hide <!--<script> \u2028"}
+ {"subject" "$1 \\ kept"}
+ ;; A placeholder in a value stays as it is.
+ {"subject" "Rename /*COUNT*/ and /*DATA*/"}]
+ "today" [{"cluster" "tardis"}]
+ "clusters" ["tardis"]
+ "retest" {"module" [1 2]}
+ "pairs" [{"module" "x.y"}]
+ "cuts" []}))))
+
+(deftest config-problems-test
+ (testing "a full config and an empty one are both usable"
+ (is (= []
+ (#'module-history/config-problems
+ {:pages [{:template "t.html", :file "page.html"}]
+ :metabase {:profile "stats", :uploads {"history.csv" 1}, :cards [2 3]}
+ :shares {"page.html" "slug-a1"}})))
+ (is (= []
+ (#'module-history/config-problems {}))))
+ (testing "each mistake is reported at the key it is under, a key's problems together"
+ (is (= [[] [:pages] [:pages] [:metabase] [:metabase] [:metabase :uploads] [:metabase :cards]]
+ (map :path (#'module-history/config-problems
+ {:share {}
+ :pages [{:template "t.html", :path "page.html"}]
+ :metabase {:uploads {"history.csv" "1"}, :cards [:a], :card 2}})))))
+ (testing "a value of the wrong kind is a problem at its key, whatever it is"
+ (are [config path] (= [path] (distinct (map :path (#'module-history/config-problems config))))
+ {:metabase 5} [:metabase]
+ {:metabase "stats"} [:metabase]
+ {:metabase {:profile "p", :uploads {:a.csv 1}}} [:metabase :uploads]
+ {:pages 5} [:pages]
+ {:pages {:file "a"}} [:pages]
+ {:shares {"a" 1}} [:shares]
+ {:shares {:a "slug"}} [:shares]
+ {:shares false} [:shares]))
+ (testing "a config that is not a map is one problem"
+ (is (= [{:path [], :message "must hold one map"}]
+ (#'module-history/config-problems []))))
+ (testing "a problem reads as its path and what is wrong there"
+ (is (= ["the config has unknown keys: :share"
+ ":pages has unknown keys: :path"
+ ":pages must be a vector of {:template \"path\", :file \"name\"}"
+ ":metabase has unknown keys: :card"
+ ":metabase :cards must be a vector of card ids"]
+ (map #'module-history/problem-line
+ (#'module-history/config-problems {:share {}
+ :pages [{:template "t.html", :path "p.html"}]
+ :metabase {:profile "p", :cards [:a], :card 1}}))))))
+
+(def ^:private native-card
+ "What `mb card get` returns for a native card the profile may change."
+ "{\"id\": 21, \"can_write\": true, \"dataset_query\": {\"database\": 1, \"stages\": [{\"native\": \"select 1\"}]}}")
+
+(defn- fake-sh
+ "A [[mage.shell/sh*]] stand-in that records each command in `calls` and answers `mb card get` with the JSON `card`.
+ A command that starts with a key of `failures` exits with that key's value."
+ [calls failures card]
+ (fn [& args]
+ (let [args (vec (if (map? (first args)) (rest args) args))]
+ (swap! calls conj (cond-> args
+ ;; The update's body is in a temporary file: record what it holds instead of its path.
+ (= ["mb" "card" "update"] (take 3 args))
+ (update 5 slurp)))
+ {:exit (or (some (fn [[start exit]] (when (= start (take (count start) args)) exit)) failures) 0)
+ :err []
+ :out (if (= ["mb" "card" "get"] (take 3 args))
+ [card]
+ [])})))
+
+(defn- publish!
+ "Publish a directory holding `files` with `config`, on a machine where `tools` are installed and `failures` are the
+ commands that fail. Returns the commands that ran with the directory's path replaced by `DIR` and the time by
+ `STAMP`, the page it wrote, its exit code, and what it printed."
+ [{:keys [config files options tools failures card]}]
+ (let [dir (fs/create-temp-dir {:prefix "module-history-test"})]
+ (doseq [[file-name content] (merge {"publish.edn" (pr-str config), "template.html" "/*COUNT*/ of /*ANCHORS*/"}
+ files)]
+ (spit (str (fs/path dir file-name)) content))
+ (let [calls (atom [])
+ exit (atom 0)
+ printed (with-out-str
+ (with-redefs [shell/sh* (fake-sh calls failures card)
+ u/can-run? (fn [tool] (contains? tools tool))]
+ (try
+ (module-history/cli-update
+ {:options (merge {:dir (str dir)
+ :publish (str (fs/path dir "publish.edn"))
+ :no-measure true}
+ options)})
+ (catch clojure.lang.ExceptionInfo e
+ (reset! exit (:babashka/exit (ex-data e)))))))]
+ (try
+ {:exit @exit
+ :calls (mapv (fn [call]
+ (mapv #(-> (str %)
+ (str/replace (str dir) "DIR")
+ (str/replace #"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ" "STAMP"))
+ call))
+ @calls)
+ :page (when (fs/exists? (fs/path dir "page.html")) (slurp (str (fs/path dir "page.html"))))
+ :printed printed}
+ (finally
+ (fs/delete-tree dir))))))
+
+(def ^:private run
+ "A publishing run that works, for [[publish!]]."
+ {:config {:pages [{:template "template.html", :file "page.html"}]
+ :metabase {:profile "stats", :uploads {"history.csv" 11, "cycles.csv" 12}, :cards [21]}
+ :shares {"page.html" "slug-a1"}}
+ ;; The template shows no retest distribution, so a `history.json` without one is enough.
+ :files {"history.csv" "sha\n"
+ "cycles.csv" "sha\n"
+ "history.json" "{\"commits\": [{}, {}], \"today\": [], \"clusters\": [\"tardis\"]}"}
+ :options {}
+ :tools #{"mb" "share"}
+ :failures {}
+ :card native-card})
+
+(defn- naming
+ "The `result` of [[publish!]] with what it printed cut down to the first match of `pattern`."
+ [pattern result]
+ (update result :printed #(re-find pattern %)))
+
+(deftest measure-test
+ (testing "measuring runs in a fresh JVM after fetching the ref it measures, when that ref is on origin"
+ (let [calls (atom [])]
+ (with-out-str
+ (with-redefs [shell/sh* (fake-sh calls {} native-card)]
+ (module-history/cli-update {:options {:dir "/tmp/out", :no-fetch true}})
+ (module-history/cli-update {:options {:dir "/tmp/out"}})
+ (module-history/cli-update {:options {:dir "/tmp/out", :ref "origin/stack-tip"}})
+ (module-history/cli-update {:options {:dir "/tmp/out", :ref "stack-tip"}})))
+ (is (= [["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""]
+ ["git" "fetch" "--quiet" "origin" "+refs/heads/master:refs/remotes/origin/master"]
+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""]
+ ["git" "fetch" "--quiet" "origin" "+refs/heads/stack-tip:refs/remotes/origin/stack-tip"]
+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""
+ ":ref" "\"origin/stack-tip\""]
+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""
+ ":ref" "\"stack-tip\""]]
+ @calls)))))
+
+(def ^:private card-get
+ "The command that fetches card 21."
+ ["mb" "card" "get" "21" "--full" "--profile" "stats" "--json" "--max-bytes" "0"])
+
+(deftest publish-test
+ (testing "pages written, cards checked, uploads replaced, cards fetched again and refreshed, shares updated"
+ (is (= {:exit 0
+ :page "2 of [\"tardis\"]"
+ :calls [card-get
+ ["mb" "upload" "replace" "12" "--file" "DIR/cycles.csv" "--profile" "stats" "--json"
+ "--max-bytes" "0"]
+ ["mb" "upload" "replace" "11" "--file" "DIR/history.csv" "--profile" "stats" "--json"
+ "--max-bytes" "0"]
+ ;; Fetched again, so an edit made during the uploads is kept.
+ card-get
+ ;; Only the query is sent, with the rest of it unchanged.
+ ["mb" "card" "update" "21" "--file"
+ (str "{\"dataset_query\":{\"database\":1,\"stages\":"
+ "[{\"native\":\"select 1\\n-- refreshed by mage module-history at STAMP\"}]}}")
+ "--profile" "stats" "--json" "--max-bytes" "0"]
+ ["share" "-e" "DIR/page.html" "slug-a1"]]}
+ (dissoc (publish! run) :printed)))))
+
+(deftest publish-dry-run-test
+ (testing "a dry run writes the page, only fetches the cards, and prints what it would run"
+ (let [{:keys [printed], :as result} (publish! (assoc run :options {:dry-run true}))]
+ (is (= {:exit 0, :page "2 of [\"tardis\"]", :calls [card-get]}
+ (dissoc result :printed)))
+ (is (= ["mb upload replace 12" "mb upload replace 11" "Would refresh card" "share -e"]
+ (re-seq #"mb upload replace \d+|Would refresh card|share -e" printed))))))
+
+(deftest publish-failing-command-test
+ (testing "a command that fails stops the run with its exit code, and nothing runs after it"
+ (is (= {:exit 3
+ :calls [card-get
+ ["mb" "upload" "replace" "12" "--file" "DIR/cycles.csv" "--profile" "stats" "--json"
+ "--max-bytes" "0"]
+ ["mb" "upload" "replace" "11" "--file" "DIR/history.csv" "--profile" "stats" "--json"
+ "--max-bytes" "0"]]}
+ (select-keys (publish! (assoc run :failures {["mb" "upload" "replace" "11"] 3})) [:exit :calls]))))
+ (testing "a card that cannot be fetched stops the run before any upload changes"
+ (is (= {:exit 4, :calls [card-get]}
+ (select-keys (publish! (assoc run :failures {["mb" "card" "get"] 4})) [:exit :calls]))))
+ (testing "a card the profile may not change stops the run before any upload changes"
+ (is (= {:exit 1, :calls [card-get]}
+ (select-keys (publish! (assoc run :card (str/replace native-card "true" "false"))) [:exit :calls])))))
+
+(deftest publish-missing-file-test
+ (testing "nothing is written or published when a file the config names is not there"
+ (is (= {:exit 1, :page nil, :calls [], :printed "cycles.csv"}
+ (naming #"cycles\.csv" (publish! (update run :files dissoc "cycles.csv")))))))
+
+(deftest publish-bad-config-test
+ (testing "a config with a mistake stops the run before anything else happens"
+ (is (= {:exit 1, :page nil, :calls [], :printed ":share"}
+ (naming #":share\b" (publish! (assoc-in run [:config :share] {})))))))
+
+(deftest publish-missing-tool-test
+ (testing "a tool the config needs and that is not installed stops the run, a dry run too"
+ (is (= {:exit 1, :page nil, :calls [], :printed "share"}
+ (naming #"share" (publish! (assoc run :tools #{"mb"}, :options {:dry-run true}))))))
+ (testing "a tool the config would not run need not be installed"
+ (is (= {:exit 0, :page "2 of [\"tardis\"]", :calls []}
+ (dissoc (publish! (assoc run
+ :config {:pages (get-in run [:config :pages])
+ :metabase {:profile "stats"}}
+ :tools #{}))
+ :printed)))))
+
+(deftest publish-stale-history-test
+ (testing "no page is written from a history.json that lacks a key its template shows"
+ (is (= {:exit 1, :page nil, :calls [], :printed "clusters"}
+ (naming #"clusters"
+ (publish! (assoc-in run [:files "history.json"] "{\"commits\": [{}, {}], \"today\": []}")))))))
diff --git a/mage/test/mage/project_tests_test.clj b/mage/test/mage/project_tests_test.clj
index dc55f7f9c9a..b64b19b54d8 100644
--- a/mage/test/mage/project_tests_test.clj
+++ b/mage/test/mage/project_tests_test.clj
@@ -44,7 +44,10 @@
(is (= [["clojure" "-M:test"]
["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"
(str '[dev.modules-config-test
+ metabase.core.module-cuts-test
metabase.core.module-cycles-test
+ metabase.core.module-history-report-test
+ metabase.core.module-history-test
metabase.core.modules-test
metabase.core.kondo-ratchet-test
metabase.core.kondo-ratchet-check-test
@@ -93,7 +96,12 @@
(deftest targeted-suites-test
(testing "modules runs only its own namespaces"
(is (= [["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"
- "[dev.modules-config-test metabase.core.module-cycles-test metabase.core.modules-test]"]]
+ (str '[dev.modules-config-test
+ metabase.core.module-cuts-test
+ metabase.core.module-cycles-test
+ metabase.core.module-history-report-test
+ metabase.core.module-history-test
+ metabase.core.modules-test])]]
(:calls (run-suites! [] ["modules"])))))
(testing "ratchets checks the repository's policy file"
(is (= [["./bin/mage" "kondo-ratchets"]]