The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add `mage module-history` to update and publish the module history (#84122)
Chris Truterb6d65f612d5c
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 12907f7fd8d..1edc2e678cb 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -1203,7 +1203,11 @@mage-namespaces{:linters- {:discouraged-java-method+ {;; mage runs under babashka, which bundles cheshire; metabase.util.json is not on its classpath.+ :discouraged-namespace+ {cheshire.core {:level :off}}++ :discouraged-java-method{java.lang.System {exit{:level :error:message "Don't use System/exit directly. Instead use mage.util/exit and let mage handle exiting the process"}}}}}}}diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex ee8c8ee3696..634e396b87f 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -85,7 +85,7 @@:config-counts {:constant-condition 1:deprecated-namespace 7:deprecated-var 16- :discouraged-namespace 14+ :discouraged-namespace 15:discouraged-var 14:inline-def 1:main-without-gen-class 1diff --git a/bb.edn b/bb.ednindex c30ea1d4c2d..26d9a220c62 100644--- a/bb.edn+++ b/bb.edn@@ -336,6 +336,22 @@:requires [[mage.modules]]:task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}+ module-history+ {:doc "Update the CSV files that track how the backend module graph changed on master, and optionally publish them. Measures only commits it has not measured before."+ :examples [["./bin/mage module-history --dir ~/module-history" "Fetch master, measure its new commits, and write the CSV files and history.json"]+ ["./bin/mage module-history --dir ~/module-history --publish ~/module-history/publish.edn" "Then fill pages, replace Metabase uploads, refresh cards and update shares, as the EDN file says"]+ ["./bin/mage module-history --dir ~/module-history-ahead --ref origin/my-stack-tip" "Write the history as it would be with a branch merged: master up to where the branch left it, then the branch's commits"]+ ["./bin/mage module-history --dir ~/module-history --no-measure --publish ~/module-history/publish.edn --dry-run" "Print what publishing the files already there would run"]]+ :options [["-d" "--dir DIR" "Directory to keep the files in (required)"]+ ["-P" "--publish FILE" "Publish the files as this EDN file says; see the mage.module-history docstring for its shape"]+ ["-D" "--dry-run" "With --publish, print the commands that would change Metabase or a share instead of running them"]+ ["-r" "--ref REF" "Measure the first-parent history of this ref instead of origin/master"]+ ["-F" "--no-fetch" "Measure the ref as last fetched"]+ ["-M" "--no-measure" "Skip fetching and measuring; use the files already in DIR"]]+ :arg-schema [:tuple]+ :requires [[mage.module-history]]+ :task (task! (mage.module-history/cli-update parsed))}+fix-modules-config{:doc "Regenerate .clj-kondo/config/modules/config.edn so it passes metabase.core.modules-test.":examples [["./bin/mage fix-modules-config" "Fix the modules config (uses the running dev REPL if available, else a JVM)"]diff --git a/dev/src/dev/module_cuts.clj b/dev/src/dev/module_cuts.cljnew file mode 100644index 00000000000..223b4d79f49--- /dev/null+++ b/dev/src/dev/module_cuts.clj@@ -0,0 +1,131 @@+(ns dev.module-cuts+ "Where the module graph could be cut next, from one commit's dependency rows.++ [[dev.module-history-report]] writes both tables for the latest commit it measures."+ (:require+ [dev.deps-graph :as deps-graph]+ [dev.module-graph :as module-graph]))++(set! *warn-on-reflection* true)++(defn upward-pairs+ "Module and ancestor pairs whose upward requires close a cycle, the pair with the most requires first.+ Each is `{:module, :ancestor, :requires, :internal}`: how many requires go from the module into that ancestor,+ and how many of those reach past the ancestor's `:api`."+ [deps modules]+ (->> (deps-graph/upward-references deps modules)+ (filter :cycle?)+ (group-by (juxt :module :ancestor))+ (map (fn [[[module ancestor] requires]]+ {:module module+ :ancestor ancestor+ :requires (count requires)+ :internal (count (remove :api? requires))}))+ (sort-by (juxt (comp - :requires) (comp str :module) (comp str :ancestor)))+ vec))++;;;; Cut candidates++(defn- requires-by-edge+ "The namespace requires behind each module edge: `{[from to] [[from-namespace to-namespace] ...]}`."+ [deps]+ (reduce (fn [edges [edge require]]+ (update edges edge (fnil conj []) require))+ {}+ (for [{:keys [namespace module], requires :deps} deps+ {to-namespace :namespace, to :module} requires+ :when (and module to (not= module to))]+ [[module to] [namespace to-namespace]])))++(defn- largest-cycle+ "The modules of the cycle of `graph` that holds the most namespaces, or nil without a cycle."+ [graph size]+ (when-let [cycles (seq (module-graph/cyclic-components graph))]+ (apply max-key size cycles)))++(defn- bottom-up+ "The modules of `graph`, one cycle, ordered so that few requires point from an earlier module to a later one.+ That is roughly the cycle as layers, the most depended-upon first.+ The `weight` of an edge is the number of requires behind it."+ [graph weight]+ ;; Eades, Lin and Smyth's greedy heuristic for a small feedback arc set: take modules that require nothing left+ ;; to the bottom and modules nothing left requires to the top, and otherwise the one that gives up the fewest+ ;; requires by going to the bottom.+ (let [requirers (reduce (fn [requirers [from to]] (update requirers to (fnil conj #{}) from))+ {}+ (for [[from tos] graph, to tos] [from to]))+ total (fn [edges] (transduce (map weight) + 0 edges))]+ (loop [left (set (keys graph))+ bottom []+ top ()]+ (if (empty? left)+ (vec (concat bottom top))+ (let [out (fn [module] (total (for [to (graph module) :when (left to)] [module to])))+ in (fn [module] (total (for [from (requirers module) :when (left from)] [from module])))+ sinks (filter #(zero? (out %)) left)+ sources (filter #(zero? (in %)) left)]+ (cond+ (seq sinks) (recur (apply disj left sinks) (into bottom (sort-by str sinks)) top)+ (seq sources) (recur (apply disj left sources) bottom (concat (sort-by str sources) top))+ :else (let [module (apply min-key #(- (out %) (in %)) (sort-by str left))]+ (recur (disj left module) (conj bottom module) top))))))))++(defn cut-candidates+ "Ways to shrink the largest module cycle by splitting it into a lower and an upper layer.+ Each candidate takes the bottom `:base-modules` of the cycle's `:cycle-modules`, in [[bottom-up]] order, as the+ lower layer and severs every require from it into the rest of the cycle:++ {:base-modules 44+ :cycle-modules 53+ :requires [[from-namespace to-namespace] ...]+ :edges [[from-module to-module requires] ...]+ :largest-after 681+ :freed 113}++ The `:edges` are the module edges behind `:requires`, the one with the most requires first.+ The `:largest-after` count is the namespaces in the largest cycle left among those modules once the requires+ are gone, and `:freed` how many fewer that is.+ Other cycles of the graph are not looked at.+ Returns the candidates of at most `max-requires` requires that no other beats on both counts, fewest requires first.+ Empty when there is no cycle."+ [deps max-requires]+ (let [requires (requires-by-edge deps)+ counts (frequencies (keep :module deps))+ size (fn [modules] (transduce (map #(counts % 0)) + 0 modules))+ graph (reduce (fn [graph [from to]] (update graph from (fnil conj #{}) to))+ (zipmap (keys counts) (repeat #{}))+ (keys requires))+ cycle (largest-cycle graph size)]+ (if-not cycle+ []+ (let [inner (into {} (map (fn [module] [module (into #{} (filter cycle) (graph module))])) cycle)+ before (size cycle)+ order (bottom-up inner #(count (requires %)))]+ (->> (for [n (range 1 (count order))+ :let [base (set (take n order))+ severed (sort (for [from base+ to (inner from)+ :when (not (base to))]+ [from to]))+ cut (into [] (mapcat #(sort (requires %))) severed)]+ :when (<= (count cut) max-requires)+ :let [after (size (largest-cycle (reduce (fn [graph [from to]] (update graph from disj to))+ inner+ severed)+ size))]]+ {:base-modules n+ :cycle-modules (count order)+ :requires cut+ :edges (->> severed+ (map (fn [edge] (conj edge (count (requires edge)))))+ (sort-by (juxt (comp - peek) first second))+ vec)+ :largest-after after+ :freed (- before after)})+ (sort-by (juxt (comp count :requires) (comp - :freed)))+ ;; Keep a candidate only when it frees more than every cheaper one.+ (reduce (fn [kept candidate]+ (if (> (:freed candidate) (:freed (peek kept) 0))+ (conj kept candidate)+ kept))+ []))))))diff --git a/dev/src/dev/module_history.clj b/dev/src/dev/module_history.cljindex ae700c7ffe8..35fd180d104 100644--- a/dev/src/dev/module_history.clj+++ b/dev/src/dev/module_history.clj@@ -554,12 +554,13 @@(defn- config-commits"Commits of `ref` since `date` that changed the module config.Before [[modules-config-path]] existed that is a change to [[legacy-config-path]], which is followed only- up to the commit that moved the config out of it, or to `ref` when `ref` predates that commit."+ up to the commit that moved the config out of it, or to `ref` when `ref` predates that commit.+ The walk starts from that commit, which is itself a config change: [[sample-commits]] removes the duplicate."[ref date](let [introduced (last (str/split-lines (git! ["log" "--first-parent" "--diff-filter=A" "--format=%H" ref"--" modules-config-path])))](concat (commits-since ref date [modules-config-path])- (commits-since (if (str/blank? introduced) ref (str introduced "^")) date [legacy-config-path]))))+ (commits-since (if (str/blank? introduced) ref introduced) date [legacy-config-path]))))(defn- last-commit-before"The last first-parent commit of `ref` before each of `dates`."@@ -573,6 +574,14 @@(let [today (LocalDate/now ZoneOffset/UTC)](take-while #(not (.isAfter ^LocalDate % today)) (iterate step start))))+(defn require-full-history!+ "Throw when the repo is a shallow clone. Its history stops at the clone's boundary, so a sample taken there would+ miss commits, or start from the wrong one, without saying so."+ []+ (when (= "true" (str/trim (git! ["rev-parse" "--is-shallow-repository"])))+ (throw (ex-info "Sampling needs the full git history, but this is a shallow clone: run `git fetch --unshallow`"+ {}))))+(defn sample-commits"Commits to measure on the first-parent history of `ref` (default `origin/master`), oldest first:@@ -583,6 +592,7 @@Dates are days in UTC."[{:keys [ref since backend-since weekly-since monthly-since extra], :or {ref "origin/master"}}]+ (require-full-history!)(let [changed (concat (when since (config-commits ref since))(when backend-since (concat (commits-since ref backend-since source-pathspecs)(config-commits ref backend-since))))diff --git a/dev/src/dev/module_history_report.clj b/dev/src/dev/module_history_report.cljnew file mode 100644index 00000000000..2f7f1277671--- /dev/null+++ b/dev/src/dev/module_history_report.clj@@ -0,0 +1,584 @@+(ns dev.module-history-report+ "The history of the backend module graph as CSV files, for charting how its cycles and boundaries changed.++ [[update!]] measures a fixed sample of master commits with [[dev.module-history]] and writes these files:++ - `history.csv`: one row per sampled commit since the first one, with cycle, exposure, `:friends` and+ upward-require counts+ - `structure.csv`: every recent backend-touching commit, thinned to the ends of each run of unchanged metrics+ - `cycles.csv`: one row per commit and module cycle, named after the cluster it is today+ - `history.json`: the commits of `history.csv` with each named cluster's size, for a page that charts them+ - `upward-pairs.csv` and `cuts.csv`: at the latest commit, the module pairs whose upward requires close a cycle,+ and the cheapest ways to split the largest cycle; see [[dev.module-cuts]]. Both are in `history.json` too.++ The sample depends only on the history of `origin/master`, so two runs at the same commit write the same files.+ A later run adds rows and keeps the earlier ones, except the tip of an earlier run, which stays only when the+ sample picks it for another reason. `./bin/mage module-history` runs it.++ This is a separate namespace because the source of [[dev.module-history]] is part of its cache key: a change to+ how rows are reported should not measure every commit again."+ (:require+ [clojure.data.csv :as csv]+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.java.shell :as sh]+ [clojure.string :as str]+ [dev.module-cuts :as module-cuts]+ [dev.module-cycle-names :as cycle-names]+ [dev.module-history :as module-history]+ [metabase.util.json :as json])+ (:import+ (java.io File)+ (java.nio.file CopyOption Files StandardCopyOption)+ (java.time Instant LocalDate OffsetDateTime)+ (java.util Locale)))++(set! *warn-on-reflection* true)++;;;; The sample++(def ^:private default-ref "origin/master")++(def ^:private history-sample+ "Commits in `history.csv`, besides the [[quarterly-commits]] before it: one a month from before the first module+ linter (#42250), one a week since that linter, and every commit that changed the module config since the dense+ sample starts."+ {:monthly-since "2023-12-01"+ :weekly-since "2024-05-08"+ :since "2025-10-01"})++(def ^:private first-quarter+ "The first quarter [[quarterly-commits]] takes the last commit before."+ "2015-04-01")++(def ^:private structure-sample+ "Commits in `structure.csv`: every one that touched the backend since the structure dashboard was first built."+ {:backend-since "2026-08-08"})++(def key-commits+ "Commits that changed the module system or the shape of the graph, by short sha. Each is measured along with its+ parent, so the step it made shows, and labelled in `history.csv`."+ {"1e9e60a0fb4" "#42250 Module linter 1.0"+ "f88dafb8be2" "#53583 Module linter 2.0"+ "734949f2676" "#65236 Halloween decoupling"+ "94d2daaafd9" "#65681 Detangling pre-Conj"+ "30616565b45" "#65144 OpenAPI→TS (regression)"+ "119487b1ccc" "#67650 Fix api→api-routes"+ "ee9ba657624" "#67292 :friends introduced"+ "c2ad36cccbd" "#71068 ee/llm into OSS"+ "ff335470e72" "#81824 Per-module db.clj"+ "eff3209a34b" "#82301 Module hierarchy"+ "4f02379e40d" "#82367 Domain subtrees"+ "30346d6a64a" "#82011 Break requiring-resolve cycles"+ "f35a8a0077b" "#77954 Untangle util"+ "ca08e86675b" "#83441 Decouple foundation modules"+ "06b78789e31" "#83453 Feature modules out of the cycle"+ "e902d2623c7" "#82386 Name module cycles"})++(def ^:private gone-clusters+ "Names for cycles that were gone before `cycle-clusters.edn` existed, so they have no anchor there: cluster name+ and a module only that cycle held."+ ;; magellanic-cloud: segments and xrays drifted beside galactic-center from #65681 until #82447 pulled them back in,+ ;; as the Magellanic Clouds are being pulled into the Milky Way.+ ;; sub-etha: the EE Metabot and Slackbot cycle, Metabot talking over Slack, named after the Hitchhiker's messaging+ ;; network to sit beside deep-thought. It lasted six days, until #71068 moved ee/llm into OSS.+ '[[magellanic-cloud xrays]+ [sub-etha enterprise/slackbot]])++;;;; Git++(defn- git-lines+ "Output lines of git run with `args`, throwing when it fails."+ [args]+ (let [{:keys [exit out err]} (apply sh/sh "git" args)]+ (when-not (zero? exit)+ (throw (ex-info (format "git %s failed: %s" (str/join " " args) err) {:args args})))+ (remove str/blank? (str/split-lines out))))++(defn- positions+ "First-parent position of every commit of `ref`, oldest first, by sha. This is `seq` in the files."+ ;; Commit timestamps carry mixed UTC offsets and two commits can share one, so rows are ordered by this and never+ ;; by timestamp.+ [ref]+ (zipmap (git-lines ["rev-list" "--first-parent" "--reverse" ref])+ (range)))++(defn- quarter-starts+ "The first day of each quarter from the ISO date `from`, which starts one, up to but not including `until`."+ [from until]+ (let [until (LocalDate/parse until)]+ (take-while #(.isBefore ^LocalDate % until)+ (iterate #(.plusMonths ^LocalDate % 3) (LocalDate/parse from)))))++(defn- quarterly-commits+ "The first commit of `ref` and the last one before each quarter until the monthly sample starts: a sparse view of+ the years before there were modules."+ [ref]+ (cons (first (git-lines ["rev-list" "--max-parents=0" "--first-parent" ref]))+ (mapcat #(git-lines ["rev-list" "-1" "--first-parent" (str "--before=" % "T00:00:00Z") ref])+ (quarter-starts first-quarter (:monthly-since history-sample)))))++(defn- key-commits-with-parents+ "Full shas of each of [[key-commits]] and its first parent."+ []+ (git-lines (into ["rev-parse"] (mapcat (juxt #(str % "^{commit}") #(str % "^"))) (keys key-commits))))++(defn- cluster-anchors+ "Cluster name -> anchor module, as `ref` has them."+ [ref]+ (->> (git-lines ["show" (str ref ":" cycle-names/clusters-file)])+ (str/join "\n")+ edn/read-string+ cycle-names/validate-anchors))++;;;; Rows++(defn- short-sha [row]+ (subs (:sha row) 0 11))++(defn- in-order+ "`rows` that have a position, one per commit, oldest first."+ [position rows]+ (->> rows+ (filter (comp position :sha))+ (into {} (map (juxt :sha identity)))+ vals+ (sort-by (comp position :sha))+ vec))++(defn- module-system?+ "Whether `row` is from a commit that has the module system."+ [row]+ (not (false? (:module-system? row))))++(defn- with-modules+ "A function that is `f` of a row with the module system, and nil of a row from before it."+ [f]+ #(when (module-system? %) (f %)))++(defn- whole-backend?+ "Whether `cycle` is the one tangle that [[as-one-tangle]] reports before the module system."+ [cycle]+ (= #{'backend} (set (:modules cycle))))++(defn- as-one-tangle+ "The `row`, with one cycle holding every namespace when it is from before the module system."+ [row]+ ;; Before the module system the whole backend is one module, which a module graph cannot show as a cycle.+ ;; Every later cycle split from that one tangle.+ (if-not (module-system? row)+ (merge row {:scc-largest-modules 1+ :scc-largest-namespaces (:namespaces row)+ :scc-cyclic-modules 1+ :scc-cyclic-namespaces (:namespaces row)+ :scc-components 1+ :scc-static-largest-namespaces (:namespaces row)+ :scc-dynamic-only-namespaces 0+ :cycles [{:modules '[backend]+ :namespaces (:namespaces row)+ :edges 0+ :density nil+ :cycle-rank 0+ :strong-bridges 0}]})+ row))++(defn- report-rows+ "The `rows` [[in-order]], each through [[as-one-tangle]] and carrying its `:position`."+ [position rows]+ (mapv #(assoc (as-one-tangle %) :position (position (:sha %))) (in-order position rows)))++(defn- cells+ "The value of each of `columns` of `[name compute]` for `row`."+ [columns row]+ (mapv (fn [[_ compute]] (compute row)) columns))++(defn- percent+ "`part` of `whole` as a percentage with one decimal, in the same form under every locale."+ [part whole]+ (String/format Locale/ROOT "%.1f" (to-array [(if (zero? whole) 0.0 (* 100.0 (/ part whole)))])))++;;;; history.csv++(def ^:private history-columns+ "Column of `history.csv` -> how to compute it from one of the [[report-rows]]."+ [[:date #(subs (:committed-at %) 0 10)]+ [:module_system module-system?]+ [:committed_at :committed-at]+ [:seq :position]+ [:sha short-sha]+ [:key_commit #(get key-commits (short-sha %) "")]+ [:namespaces :namespaces]+ [:modules :modules]+ [:top_level_modules :top-level-modules]+ [:nested_modules :nested-modules]+ [:scc_largest_modules :scc-largest-modules]+ [:scc_largest_namespaces :scc-largest-namespaces]+ [:scc_largest_pct #(percent (:scc-largest-namespaces %) (:namespaces %))]+ [:scc_cyclic_modules :scc-cyclic-modules]+ [:scc_cyclic_namespaces :scc-cyclic-namespaces]+ [:scc_cyclic_pct #(percent (:scc-cyclic-namespaces %) (:namespaces %))]+ [:scc_components :scc-components]+ [:scc_static_largest_namespaces :scc-static-largest-namespaces]+ [:scc_dynamic_only_namespaces :scc-dynamic-only-namespaces]+ [:module_edges :module-edges]+ [:top_level_edges :top-level-edges]+ [:upward_requires :upward-requires]+ [:upward_module_edges :upward-module-edges]+ [:upward_internal_requires :upward-internal-requires]+ [:upward_cycle_requires :upward-cycle-requires]+ ;; With one module there are no boundaries, so exposure means nothing before #42250.+ [:exposed_namespaces (with-modules :exposed-namespaces)]+ [:exposed_pct (with-modules #(percent (:exposed-namespaces %) (:namespaces %)))]+ [:hidden_modules :hidden-modules]+ [:hidden_namespaces :hidden-namespaces]+ [:cross_top_level_required_namespaces :cross-top-level-required-namespaces]+ [:api_any_modules :api-any-modules]+ [:api_any_namespaces :api-any-namespaces]+ [:uses_any_modules :uses-any-modules]+ [:friend_grants :friend-grants]+ [:cross_top_level_friend_grants :cross-top-level-friend-grants]+ [:friend_exposed_namespaces :friend-exposed-namespaces]+ [:friend_access_paths :friend-access-paths]+ [:largest_ns_level_scc :largest-ns-level-scc]+ [:mutual_dependency_pairs :mutual-dependency-pairs]+ [:non_api_cross_module_requires :non-api-cross-module-requires]+ [:friend_internal_requires :friend-internal-requires]+ [:subject :subject]])++(defn- table+ "A header and one line per row, from `columns` of `[name compute]`."+ [columns rows]+ (cons (mapv (comp name first) columns)+ (map #(cells columns %) rows)))++(defn history-table+ "Header and lines of `history.csv` for measured `rows`. `position` gives a commit's place on master by sha; a row+ without one is left out."+ [position rows]+ (table history-columns (report-rows position rows)))++;;;; structure.csv++(defn- rounded-percent [part whole]+ (if (zero? whole) 0.0 (/ (Math/round (* 1000.0 (/ part whole))) 10.0)))++(def ^:private structure-id-columns+ [[:committed_at :committed-at]+ [:sha :sha]+ [:pr #(some->> (re-find #"\(#(\d+)\)\s*$" (:subject %)) second)]+ [:subject :subject]])++(def ^:private structure-metric-columns+ [[:modules :configured-modules]+ [:top_level_modules :configured-top-level-modules]+ [:namespaces :distinct-namespaces]+ [:module_edges :module-edges]+ [:cyclic_clusters :scc-components]+ [:scc1_modules :scc1-modules]+ [:scc1_namespaces :scc1-namespaces]+ [:scc2_modules :scc2-modules]+ [:scc2_namespaces :scc2-namespaces]+ [:scc3_modules :scc3-modules]+ [:scc3_namespaces :scc3-namespaces]+ [:modules_in_cycles :scc-cyclic-modules]+ [:namespaces_in_cycles :scc-cyclic-namespaces]+ [:pct_namespaces_in_cycles #(rounded-percent (:scc-cyclic-namespaces %) (:distinct-namespaces %))]+ [:largest_ns_level_scc :largest-ns-level-scc]+ [:mutual_dependency_pairs :mutual-dependency-pairs]+ [:api_any :api-any-modules]+ [:uses_any :uses-any-modules]+ [:friend_edges :friend-grants]+ [:ns_prefixes :ns-prefixes]+ [:model_imports_bypass :model-imports-bypass]+ [:friend_internal_namespaces :friend-internal-namespaces]+ [:friend_internal_requires :friend-internal-requires]+ [:non_api_cross_module_requires :non-api-cross-module-requires]+ [:total_api_namespaces :total-api-namespaces]+ [:legacy_rest_modules :legacy-rest-modules]+ [:upward_requires :upward-requires]+ [:upward_cycle_requires :upward-cycle-requires]+ [:upward_internal_requires :upward-internal-requires]+ [:upward_module_edges :upward-module-edges]+ [:exposed_namespaces :exposed-namespaces]+ [:cross_top_level_required_namespaces :cross-top-level-required-namespaces]])++(defn- ends-of-unchanged-runs+ "The rows whose `values` differ from either neighbour's, plus the first and last: both ends of every run of+ commits that changed nothing."+ [values rows]+ (let [rows (vec rows)+ values (mapv values rows)+ n (count rows)]+ (for [i (range n)+ :when (or (zero? i)+ (= i (dec n))+ (not= (values i) (values (dec i)))+ (not= (values i) (values (inc i))))]+ (rows i))))++(defn structure-table+ "Header and lines of `structure.csv` for measured `rows`: most commits move no metric, so each run of unchanged+ commits keeps only its first and last."+ [position rows]+ (table (concat structure-id-columns structure-metric-columns)+ (ends-of-unchanged-runs #(cells structure-metric-columns %) (in-order position rows))))++;;;; cycles.csv++(defn- cluster-names+ "Cluster names of `anchors`, the one whose cycle has the most namespaces at `latest-row` first. A cluster that is+ no longer a cycle there sorts last."+ [anchors latest-row]+ (let [size (fn [cluster]+ (or (some (fn [cycle]+ (when (some #{(anchors cluster)} (:modules cycle))+ (:namespaces cycle)))+ (:cycles latest-row))+ 0))]+ (sort-by (juxt (comp - size) str) (keys anchors))))++(defn- cluster-name+ "The first of `names` whose anchor `cycle` holds, else its [[gone-clusters]] entry, else `unnamed`.+ A cycle that holds several anchors is those clusters before they split, and it takes the name of the largest.+ The one tangle from before the module system takes the first of `names`."+ [anchors names cycle]+ (let [members (set (:modules cycle))]+ (or (when (whole-backend? cycle) (str (first names)))+ (some-> (first (filter #(contains? members (anchors %)) names)) str)+ (some (fn [[cluster anchor]] (when (contains? members anchor) (str cluster))) gone-clusters)+ "unnamed")))++(defn- instant ^Instant [row]+ (.toInstant (OffsetDateTime/parse (:committed-at row))))++(defn- last-commit-of-each-instant+ "The `rows` with only the last of any commits that share an instant."+ [rows]+ ;; A merge queue can land two commits in the same second, and a chart stacked over time would add them together.+ (map last (partition-by instant rows)))++(defn cycles-table+ "Header and lines of `cycles.csv`: one line for each module cycle at each of `rows`, named by `anchors`, a map of+ cluster name to anchor module. The `anchors` column counts the anchors a cycle holds, so a merge of today's+ clusters shows."+ [anchors position rows]+ (let [rows (last-commit-of-each-instant (report-rows position rows))+ names (cluster-names anchors (last rows))]+ (cons ["committed_at" "seq" "sha" "cluster" "anchors" "modules" "namespaces" "edges" "density" "cycle_rank"+ "strong_bridges" "members" "subject"]+ (for [row rows+ cycle (:cycles row)+ :let [members (set (:modules cycle))]]+ [(:committed-at row)+ (:position row)+ (short-sha row)+ (cluster-name anchors names cycle)+ (if (whole-backend? cycle)+ (count anchors)+ (count (filter (comp members anchors) names)))+ (count members)+ (:namespaces cycle)+ (:edges cycle)+ (:density cycle)+ (:cycle-rank cycle)+ (:strong-bridges cycle)+ (str/join " " (:modules cycle))+ (:subject row)]))))++;;;; history.json++(defn- cycle-holding+ "The cycle at `row` that holds `module`. Before the module system that is the one tangle, whatever the module."+ [row module]+ (first (filter #(or (whole-backend? %) (some #{module} (:modules %))) (:cycles row))))++(def ^:private commit-keys+ "Key of a commit in `history.json` -> how to compute it from one of the [[report-rows]] that carries its+ `:largest-cycle` and `:lanes`. Page templates kept outside this repository read these keys by name."+ [;; When it was committed, with its UTC offset.+ [:at :committed-at]+ ;; The day it was committed.+ [:date #(subs (:committed-at %) 0 10)]+ ;; Short sha.+ [:sha short-sha]+ ;; Label of one of the [[key-commits]], else an empty string.+ [:key #(get key-commits (short-sha %) "")]+ ;; First line of the commit message.+ [:subject :subject]+ ;; Backend namespaces.+ [:ns :namespaces]+ ;; Namespaces in the largest cycle.+ [:scc :scc-largest-namespaces]+ ;; Modules in the largest cycle.+ [:sccm :scc-largest-modules]+ ;; Share of the namespaces that the largest cycle holds, as a percentage.+ [:pct #(parse-double (percent (:scc-largest-namespaces %) (:namespaces %)))]+ ;; Namespaces in any cycle.+ [:cyc :scc-cyclic-namespaces]+ ;; Namespaces that only dynamic requires keep in the largest cycle.+ [:dyn :scc-dynamic-only-namespaces]+ ;; Strong bridges of the largest cycle.+ [:bridges (comp :strong-bridges :largest-cycle)]+ ;; Cycle rank of the largest cycle.+ [:rank (comp :cycle-rank :largest-cycle)]+ ;; Mean share of the backend a change retests, selecting tests by module.+ [:rtm :retest-module-mean]+ ;; The same, selecting tests by namespace.+ [:rtn :retest-namespace-mean]+ ;; Leaf modules, which nothing else requires. Before modules existed the one module is not a leaf worth counting.+ [:lfm (with-modules :leaf-modules)]+ ;; Namespaces in leaf modules.+ [:lfmn (with-modules :leaf-module-namespaces)]+ ;; Leaf namespaces, which nothing else requires.+ [:lfn :leaf-namespaces]+ ;; Requires from a module into one of its ancestors.+ [:up :upward-requires]+ ;; Those of them that close a cycle.+ [:upcyc :upward-cycle-requires]+ ;; Those of them that reach past the ancestor's API.+ [:upint :upward-internal-requires]+ ;; Namespaces another top-level module may require. With one module there are no boundaries to expose them.+ [:exp (with-modules :exposed-namespaces)]+ ;; Non-API namespaces that a `:friends` grant opens to another top-level module.+ [:fx :friend-exposed-namespaces]+ ;; Non-API namespaces that a friend really requires.+ [:fxu :friend-internal-namespaces]+ ;; Namespaces required from another top-level module.+ [:xreq :cross-top-level-required-namespaces]+ ;; Top-level modules.+ [:top :top-level-modules]+ ;; Nested modules.+ [:nested :nested-modules]+ ;; Namespaces in the largest cycle of the namespace graph.+ [:nslvl :largest-ns-level-scc]+ ;; Pairs of modules that require each other, leaving out a module and its ancestor.+ [:mutual :mutual-dependency-pairs]+ ;; Requires of another module's non-API namespace.+ [:nonapi :non-api-cross-module-requires]+ ;; Cluster name -> the `[namespaces label]` of the cycle that holds its anchor, `[0 nil]` when there is none.+ [:lanes :lanes]])++(defn page-data+ "What a page needs to chart measured `rows`, as a map ready for JSON:++ - `:clusters`: every cluster name, today's largest first, then those that are gone+ - `:commits`: one map per commit, under the short keys that [[commit-keys]] lists+ - `:today`: the cycles at the latest commit, largest first+ - `:retest`: at the latest commit, the namespaces in each tenth of retest share, selecting tests by `:module`+ and by `:namespace`"+ [anchors position rows]+ (let [rows (report-rows position rows)+ names (cluster-names anchors (last rows))+ named (concat (map (juxt str anchors) names)+ (map (fn [[cluster anchor]] [(str cluster) anchor]) gone-clusters))+ lanes (fn [row]+ (into {}+ (for [[cluster anchor] named+ :let [cycle (cycle-holding row anchor)]]+ [cluster (if cycle [(:namespaces cycle) (cluster-name anchors names cycle)] [0 nil])])))+ ;; The cycle with the most namespaces, which the `:scc-largest-*` counts describe.+ largest #(when-let [cycles (seq (:cycles %))] (apply max-key :namespaces cycles))]+ {:clusters (mapv first named)+ :commits (for [row rows]+ (zipmap (map first commit-keys)+ (cells commit-keys (assoc row :largest-cycle (largest row), :lanes (lanes row)))))+ :retest (:retest-histogram (last rows))+ :today (for [cycle (sort-by (comp - :namespaces) (:cycles (last rows)))]+ {:cluster (cluster-name anchors names cycle)+ :modules (count (:modules cycle))+ :namespaces (:namespaces cycle)+ :edges (:edges cycle)+ :density (:density cycle)+ :rank (:cycle-rank cycle)+ :bridges (:strong-bridges cycle)+ :members (str/join " " (sort (map str (:modules cycle))))})}))++;;;; Files++(defn- write-atomically!+ "Write `file` with `write`, a function of a writer, through a temporary file, so a reader never sees half a file."+ [^File file write]+ (io/make-parents file)+ (let [tmp (File/createTempFile (.getName file) ".tmp" (.getParentFile file))]+ (with-open [writer (io/writer tmp)]+ (write writer))+ (Files/move (.toPath tmp) (.toPath file)+ (into-array CopyOption [StandardCopyOption/ATOMIC_MOVE StandardCopyOption/REPLACE_EXISTING]))))++(defn- write-csv!+ "Write `lines` to `file` and return how many follow the header. Nil is an empty cell."+ [file lines]+ (write-atomically! file #(csv/write-csv % (map (fn [line] (map (fn [cell] (if (nil? cell) "" (str cell))) line))+ lines)))+ (dec (count lines)))++;;;; upward-pairs.csv and cuts.csv++(def ^:private cut-require-limit+ "A cut that severs more requires than this is not offered."+ 30)++(defn tip-tables+ "The two tables about one commit's `deps` and `modules`, as maps ready for JSON:++ - `:pairs`: [[module-cuts/upward-pairs]], with names as strings+ - `:cuts`: [[module-cuts/cut-candidates]], each as `{:base, :of, :requires, :after, :pct, :freed, :edges,+ :severed}`, where `:pct` is `:after` as a share of all namespaces, `:edges` has names as strings and+ `:severed` lists each require as `from-namespace -> to-namespace`"+ [deps modules]+ {:pairs (for [pair (module-cuts/upward-pairs deps modules)]+ (-> pair (update :module str) (update :ancestor str)))+ :cuts (for [{:keys [base-modules cycle-modules requires edges largest-after freed]}+ (module-cuts/cut-candidates deps cut-require-limit)]+ {:base base-modules+ :of cycle-modules+ :requires (count requires)+ :after largest-after+ :pct (rounded-percent largest-after (count deps))+ :freed freed+ :edges (mapv (fn [[from to n]] [(str from) (str to) n]) edges)+ :severed (mapv (fn [[from to]] (str from " -> " to)) requires)})})++(defn- pairs-table [sha pairs]+ (cons ["sha" "module" "ancestor" "requires" "internal"]+ (for [{:keys [module ancestor requires internal]} pairs]+ [sha module ancestor requires internal])))++(defn- cuts-table [sha cuts]+ (cons ["sha" "base_modules" "cycle_modules" "requires" "largest_cycle_after" "share_after_pct" "namespaces_freed"+ "severed"]+ (for [{:keys [base of requires after pct freed severed]} cuts]+ [sha base of requires after pct freed (str/join "; " severed)])))++(defn update!+ "Measure the sample on `:ref` (default `origin/master`, as last fetched) and write `history.csv`,+ `structure.csv`, `cycles.csv`, `upward-pairs.csv`, `cuts.csv` and `history.json` into `:dir`.+ Commits measured before come from the cache.++ clojure -X:dev dev.module-history-report/update! :dir '\"target/module-history\"'"+ [{:keys [dir ref], :or {ref default-ref}}]+ (when (str/blank? (str dir))+ (throw (ex-info "update! needs :dir, the directory to write the files into" {})))+ ;; Before any step that reads the history, so a shallow clone fails here and not on a missing parent.+ (module-history/require-full-history!)+ (let [dir (io/file (str dir))+ ;; Every step reads this one commit, so a fetch during a long run cannot mix two tips in the files.+ tip (first (git-lines ["rev-parse" (str ref "^{commit}")]))+ history (module-history/backfill! (assoc history-sample+ :ref tip+ :extra (concat (quarterly-commits tip) (key-commits-with-parents))))+ structure (module-history/backfill! (assoc structure-sample :ref tip))+ position (positions tip)+ anchors (cluster-anchors tip)+ sha (subs tip 0 11)+ tables (let [{:keys [deps modules]} (module-history/snapshot tip)] (tip-tables deps modules))+ page (merge (page-data anchors position history) tables)]+ (doseq [[file-name lines] [["history.csv" (history-table position history)]+ ["structure.csv" (structure-table position structure)]+ ["cycles.csv" (cycles-table anchors position (concat history structure))]+ ["upward-pairs.csv" (pairs-table sha (:pairs tables))]+ ["cuts.csv" (cuts-table sha (:cuts tables))]]]+ (println (format "%-16s %5d rows" file-name (write-csv! (io/file dir file-name) lines))))+ (write-atomically! (io/file dir "history.json") #(json/encode-to page % {}))+ (println (format "%-16s %5d commits" "history.json" (count (:commits page))))+ (println "Written to" (.getCanonicalPath dir))))diff --git a/dev/test/metabase/core/module_cuts_test.clj b/dev/test/metabase/core/module_cuts_test.cljnew file mode 100644index 00000000000..d4d13427fb1--- /dev/null+++ b/dev/test/metabase/core/module_cuts_test.clj@@ -0,0 +1,105 @@+(ns metabase.core.module-cuts-test+ (:require+ [clojure.test :refer :all]+ [dev.module-cuts :as module-cuts]))++(defn- row+ "A dependency row for `namespace` in `module` that requires each `[namespace module]` of `requires`."+ [namespace module requires]+ {:namespace namespace+ :module module+ :deps (for [[required-namespace required-module] requires]+ {:namespace required-namespace, :module required-module})})++(deftest ^:parallel upward-pairs-test+ ;; x requires its children x.y and x.z, so their requires back into x close a cycle. x.w requires x, and+ ;; nothing requires x.w.+ (let [modules '{x {}, x.y {}, x.z {}, x.w {}}+ deps [(row 'metabase.x.core 'x '[[metabase.x.y.core x.y] [metabase.x.z.core x.z]])+ (row 'metabase.x.impl 'x [])+ (row 'metabase.x.y.core 'x.y '[[metabase.x.core x] [metabase.x.impl x]])+ (row 'metabase.x.y.other 'x.y '[[metabase.x.impl x]])+ (row 'metabase.x.z.core 'x.z '[[metabase.x.core x]])+ (row 'metabase.x.w.core 'x.w '[[metabase.x.impl x]])]]+ (testing "pairs that close a cycle are counted, with the requires that reach past the ancestor's API"+ (is (= '[{:module x.y, :ancestor x, :requires 3, :internal 2}+ {:module x.z, :ancestor x, :requires 1, :internal 0}]+ (module-cuts/upward-pairs deps modules))))))++(deftest ^:parallel cut-candidates-test+ ;; A cycle of base, mid and top, three namespaces each. One require goes up from base into mid, and two go up+ ;; from mid into top. An unrelated module requires base, and base.a requires inside its own module.+ (let [deps [(row 'metabase.base.core 'base '[[metabase.mid.core mid]])+ (row 'metabase.base.a 'base '[[metabase.base.core base]])+ (row 'metabase.base.b 'base [])+ (row 'metabase.mid.core 'mid '[[metabase.base.core base] [metabase.top.core top]])+ (row 'metabase.mid.a 'mid '[[metabase.top.core top]])+ (row 'metabase.mid.b 'mid [])+ (row 'metabase.top.core 'top '[[metabase.mid.core mid] [metabase.base.core base]])+ (row 'metabase.top.a 'top [])+ (row 'metabase.top.b 'top [])+ (row 'metabase.other.core 'other '[[metabase.base.core base]])]]+ (testing "a bottom layer is offered when cutting it loose frees more than every cheaper cut, cheapest first"+ ;; The order is base, top, mid. One require frees base; two leave no cycle at all.+ (is (= '[{:base-modules 1+ :cycle-modules 3+ :requires [[metabase.base.core metabase.mid.core]]+ :edges [[base mid 1]]+ :largest-after 6+ :freed 3}+ {:base-modules 2+ :cycle-modules 3+ :requires [[metabase.base.core metabase.mid.core]+ [metabase.top.core metabase.mid.core]]+ :edges [[base mid 1] [top mid 1]]+ :largest-after 0+ :freed 9}]+ (module-cuts/cut-candidates deps 30))))+ (testing "the cycle with the most namespaces is the one cut, not the one with the most modules"+ (is (= [{:cycle-modules 2, :edges '[[q p 1]], :largest-after 0, :freed 10}]+ (map #(select-keys % [:cycle-modules :edges :largest-after :freed])+ (module-cuts/cut-candidates (into deps+ (concat [(row 'metabase.p.core 'p '[[metabase.q.core q]])+ (row 'metabase.q.core 'q '[[metabase.p.core p]])]+ (for [n (range 8)]+ (row (symbol (str "metabase.p.n" n)) 'p []))))+ 30)))))+ (testing "another cycle in the graph does not count as what is left of this one"+ (is (= [[1 6] [2 0]]+ (map (juxt :base-modules :largest-after)+ (module-cuts/cut-candidates (concat deps+ [(row 'metabase.p.core 'p '[[metabase.q.core q]])+ (row 'metabase.p.a 'p [])+ (row 'metabase.q.core 'q '[[metabase.p.core p]])+ (row 'metabase.q.a 'q [])])+ 30)))))+ (testing "a layer that costs as much as another and frees less is dropped, and the cheapest comes first"+ ;; With far on top the order is top, mid, base, far. The bottom two cost three requires, as the bottom one+ ;; does, and free less; the bottom three cost one. Requires are listed by module edge, then by namespace.+ (is (= '[{:base-modules 3+ :requires [[metabase.top.c metabase.far.core]]+ :freed 1}+ {:base-modules 1+ :requires [[metabase.top.core metabase.base.core]+ [metabase.top.c metabase.far.core]+ [metabase.top.core metabase.mid.core]]+ :freed 5}]+ (map #(select-keys % [:base-modules :requires :freed])+ (module-cuts/cut-candidates+ (concat deps+ [(row 'metabase.far.core 'far '[[metabase.top.core top]+ [metabase.top.a top]+ [metabase.mid.core mid]])+ (row 'metabase.top.c 'top '[[metabase.far.core far]])])+ 30)))))+ (testing "a require inside one module is not an edge"+ (is (not-any? (fn [[from to]] (= from to))+ (keys (#'module-cuts/requires-by-edge deps)))))+ (testing "a candidate needing more requires than the limit is left out"+ (is (= [1]+ (map :base-modules (module-cuts/cut-candidates deps 1)))))+ (testing "without a cycle there is nothing to cut"+ (is (= []+ (module-cuts/cut-candidates [(row 'metabase.base.core 'base [])+ (row 'metabase.top.core 'top '[[metabase.base.core base]])]+ 30))))))diff --git a/dev/test/metabase/core/module_history_report_test.clj b/dev/test/metabase/core/module_history_report_test.cljnew file mode 100644index 00000000000..dc3fb944b8d--- /dev/null+++ b/dev/test/metabase/core/module_history_report_test.clj@@ -0,0 +1,328 @@+(ns metabase.core.module-history-report-test+ (:require+ [clojure.test :refer :all]+ [dev.module-history-report :as report]+ [metabase.util.json :as json]))++(defn- lines->maps+ "Each line after the header as a map of column name to value."+ [[header & lines]]+ (mapv #(zipmap (map keyword header) %) lines))++(def ^:private position+ "Places on master for the commits these tests use. `off-master` has none."+ {"aaaaaaaaaaaa1" 10+ "bbbbbbbbbbbb2" 11+ "cccccccccccc3" 12+ "dddddddddddd4" 13})++(deftest ^:parallel quarter-starts-test+ (testing "a quarter that starts on the end date is left out"+ (is (= ["2015-04-01" "2015-07-01" "2015-10-01" "2016-01-01"]+ (map str (#'report/quarter-starts "2015-04-01" "2016-04-01")))))+ (testing "a quarter that starts before the end date is kept"+ (is (= ["2015-04-01" "2015-07-01"]+ (map str (#'report/quarter-starts "2015-04-01" "2015-07-02"))))))++(deftest ^:parallel history-table-test+ (let [rows [{:sha "bbbbbbbbbbbb2"+ :committed-at "2026-01-02T10:00:00Z"+ :subject "Carve \"x\", then y (#2)"+ :module-system? true+ :namespaces 200+ :scc-largest-namespaces 50+ :scc-cyclic-namespaces 75+ :exposed-namespaces 30}+ ;; Later on the clock than the commit above, but earlier on master.+ {:sha "aaaaaaaaaaaa1"+ :committed-at "2026-01-02T23:00:00+13:00"+ :subject "Before the module system"+ :module-system? false+ :namespaces 120}+ {:sha "off-master"+ :committed-at "2026-01-03T00:00:00Z"+ :subject "On a branch"+ :namespaces 1}]]+ (testing "rows follow master, not the clock, and a commit that is not on master is left out"+ (is (=? [;; Before the module system the backend is one tangle of every namespace, with no exposure to report.+ {:seq 10+ :sha "aaaaaaaaaaa"+ :date "2026-01-02"+ :subject "Before the module system"+ :module_system false+ :scc_largest_modules 1+ :scc_largest_namespaces 120+ :scc_largest_pct "100.0"+ :scc_cyclic_pct "100.0"+ :scc_components 1+ :exposed_namespaces nil+ :exposed_pct nil}+ ;; Percentages have one decimal.+ {:seq 11+ :sha "bbbbbbbbbbb"+ :date "2026-01-02"+ :subject "Carve \"x\", then y (#2)"+ :module_system true+ :scc_largest_pct "25.0"+ :scc_cyclic_pct "37.5"+ :exposed_pct "15.0"}]+ (lines->maps (report/history-table position rows)))))))++(deftest ^:parallel key-commit-label-test+ (let [[sha label] (first report/key-commits)]+ (is (=? [{:key_commit label} {:key_commit ""}]+ (lines->maps (report/history-table {(str sha "0000") 1, "bbbbbbbbbbbb2" 2}+ (for [commit [(str sha "0000") "bbbbbbbbbbbb2"]]+ {:sha commit+ :committed-at "2026-01-01T00:00:00Z"+ :namespaces 10+ :scc-largest-namespaces 0+ :scc-cyclic-namespaces 0+ :exposed-namespaces 0})))))))++(deftest ^:parallel structure-table-test+ (let [row (fn [sha subject namespaces]+ {:sha sha+ :committed-at "2026-09-01T00:00:00Z"+ :subject subject+ :distinct-namespaces namespaces+ :scc-cyclic-namespaces 0})+ rows [(row "aaaaaaaaaaaa1" "Start a run (#1)" 100)+ (row "bbbbbbbbbbbb2" "Change nothing (#2)" 100)+ (row "cccccccccccc3" "End the run" 100)+ (row "dddddddddddd4" "Add a namespace (#4)" 101)]]+ (testing "a run of commits that move no metric keeps its two ends, and the PR number comes from the subject"+ (is (=? [{:sha "aaaaaaaaaaaa1", :pr "1", :namespaces 100}+ {:sha "cccccccccccc3", :pr nil, :namespaces 100}+ {:sha "dddddddddddd4", :pr "4", :namespaces 101}]+ (lines->maps (report/structure-table position (reverse rows))))))))++(deftest ^:parallel cycles-table-test+ (let [anchors '{galactic-center query-processor+ deep-thought metabot+ tardis lib}+ cycle (fn [modules namespaces]+ {:modules modules+ :namespaces namespaces+ :edges 9+ :density 0.5+ :cycle-rank 2+ :strong-bridges 1})+ rows [{:sha "aaaaaaaaaaaa1"+ :committed-at "2026-01-01T00:00:00Z"+ :subject "No modules yet"+ :module-system? false+ :namespaces 700}+ {:sha "bbbbbbbbbbbb2"+ :committed-at "2026-01-02T00:00:00Z"+ :subject "One big cycle"+ :cycles [(cycle '[lib metabot query-processor] 600)+ (cycle '[segments xrays] 20)+ (cycle '[a b] 5)]}+ ;; The same second as the commit after it, written with another offset.+ {:sha "cccccccccccc3"+ :committed-at "2026-01-03T02:00:00+02:00"+ :subject "Dropped"+ :cycles [(cycle '[lib metabot query-processor] 600)]}+ {:sha "dddddddddddd4"+ :committed-at "2026-01-03T00:00:00Z"+ :subject "Split"+ ;; metabot's cycle is the largest at the latest commit, so its name goes first in a merge.+ :cycles [(cycle '[metabot slackbot] 300)+ (cycle '[lib query-processor] 200)]}]]+ (is (=? [{:seq 10+ :cluster "deep-thought"+ :anchors 3+ :modules 1+ :namespaces 700+ :members "backend"}+ {:seq 11+ :cluster "deep-thought"+ :anchors 3+ :modules 3+ :namespaces 600+ :members "lib metabot query-processor"+ :density 0.5+ :cycle_rank 2+ :strong_bridges 1+ :edges 9}+ {:seq 11+ :cluster "magellanic-cloud"+ :anchors 0+ :members "segments xrays"}+ {:seq 11+ :cluster "unnamed"+ :anchors 0+ :members "a b"}+ {:seq 13+ :cluster "deep-thought"+ :anchors 1+ :sha "ddddddddddd"+ :subject "Split"}+ ;; tardis ties with galactic-center on this cycle's size, and the names then sort by letter.+ {:seq 13+ :cluster "galactic-center"+ :anchors 2+ :members "lib query-processor"}]+ (lines->maps (report/cycles-table anchors position rows))))))++(def ^:private page-anchors+ '{galactic-center query-processor+ tardis lib})++(def ^:private page-rows+ "A commit from before the module system and one after it, for [[report/page-data]]."+ (let [counts {:namespaces 100+ :scc-largest-namespaces 60+ :exposed-namespaces 40+ :retest-module-mean 70.0+ :retest-namespace-mean 20.0+ :retest-histogram {:module [1 0 0 0 0 0 0 0 0 9], :namespace [9 1]}+ :leaf-modules 3+ :leaf-module-namespaces 12+ :leaf-namespaces 30+ :friend-internal-namespaces 4}]+ [(merge counts {:sha "aaaaaaaaaaaa1"+ :committed-at "2026-01-01T00:00:00Z"+ :subject "No modules yet"+ :module-system? false})+ (merge counts {:sha "bbbbbbbbbbbb2"+ :committed-at "2026-01-02T00:00:00Z"+ :subject "Split (#2)"+ :cycles [{:modules '[lib lib.schema]+ :namespaces 20+ :density 0.5+ :cycle-rank 1+ :strong-bridges 2+ :edges 2}+ {:modules '[query-processor b a]+ :namespaces 60+ :density 0.4+ :cycle-rank 3+ :strong-bridges 0+ :edges 5}]})]))++(deftest ^:parallel page-data-test+ (is (=? {:clusters ["galactic-center" "tardis" "magellanic-cloud" "sub-etha"]+ :retest {:module [1 0 0 0 0 0 0 0 0 9], :namespace [9 1]}+ :commits [{:sha "aaaaaaaaaaa"+ :pct 100.0+ :lfm nil+ :lfn 30+ :exp nil+ :bridges 0+ :rank 0+ ;; Every cluster was inside the one tangle.+ :lanes {"galactic-center" [100 "galactic-center"]+ "tardis" [100 "galactic-center"]+ "magellanic-cloud" [100 "galactic-center"]}}+ {:sha "bbbbbbbbbbb"+ :date "2026-01-02"+ :subject "Split (#2)"+ :ns 100+ :scc 60+ :pct 60.0+ :rtm 70.0+ :rtn 20.0+ :fxu 4+ :lfm 3+ :lfmn 12+ :lfn 30+ :exp 40+ ;; Of the cycle with the most namespaces.+ :bridges 0+ :rank 3+ :lanes {"galactic-center" [60 "galactic-center"]+ "tardis" [20 "tardis"]+ "magellanic-cloud" [0 nil]+ "sub-etha" [0 nil]}}]+ :today [{:cluster "galactic-center"+ :modules 3+ :namespaces 60+ :members "a b query-processor"+ :rank 3+ :bridges 0}+ {:cluster "tardis"+ :modules 2+ :namespaces 20+ :members "lib lib.schema"}]}+ (report/page-data page-anchors position page-rows))))++(deftest ^:parallel page-data-json-test+ (testing "the page data has exactly the keys page templates read, before the tip tables are added"+ (let [page (json/decode (json/encode (report/page-data page-anchors position page-rows)))]+ (is (= {:page #{"clusters" "commits" "retest" "today"}+ :commit #{"at"+ "bridges"+ "cyc"+ "date"+ "dyn"+ "exp"+ "fx"+ "fxu"+ "key"+ "lanes"+ "lfm"+ "lfmn"+ "lfn"+ "mutual"+ "nested"+ "nonapi"+ "ns"+ "nslvl"+ "pct"+ "rank"+ "rtm"+ "rtn"+ "scc"+ "sccm"+ "sha"+ "subject"+ "top"+ "up"+ "upcyc"+ "upint"+ "xreq"}}+ {:page (set (keys page))+ :commit (set (keys (first (get page "commits"))))})))))++(deftest ^:parallel tip-tables-test+ ;; x requires its child x.y, which requires x's API and one internal namespace back. Cutting the one require+ ;; from x into x.y leaves no cycle.+ (let [dep (fn [namespace module] {:namespace namespace, :module module})+ deps [{:namespace 'metabase.x.core, :module 'x, :deps [(dep 'metabase.x.y.core 'x.y)]}+ {:namespace 'metabase.x.impl, :module 'x, :deps []}+ {:namespace 'metabase.x.y.core+ :module 'x.y+ :deps [(dep 'metabase.x.core 'x) (dep 'metabase.x.impl 'x)]}+ {:namespace 'metabase.z.core, :module 'z, :deps []}]]+ (is (= {:pairs [{:module "x.y", :ancestor "x", :requires 2, :internal 1}]+ :cuts [{:base 1+ :of 2+ :requires 1+ :after 0+ :pct 0.0+ :freed 3+ :edges [["x" "x.y" 1]]+ :severed ["metabase.x.core -> metabase.x.y.core"]}]}+ (report/tip-tables deps '{x {}, x.y {}, z {}})))))++(deftest ^:parallel tip-table-columns-test+ (testing "the two tip tables keep the columns their uploads were created with"+ (is (= [{:sha "abc", :module "x.y", :ancestor "x", :requires 2, :internal 1}]+ (lines->maps (#'report/pairs-table "abc" [{:module "x.y", :ancestor "x", :requires 2, :internal 1}]))))+ (is (= [{:sha "abc"+ :base_modules 1+ :cycle_modules 2+ :requires 2+ :largest_cycle_after 0+ :share_after_pct 0.0+ :namespaces_freed 3+ :severed "a.core -> b.core; a.impl -> b.core"}]+ (lines->maps (#'report/cuts-table "abc" [{:base 1+ :of 2+ :requires 2+ :after 0+ :pct 0.0+ :freed 3+ :severed ["a.core -> b.core" "a.impl -> b.core"]}]))))))diff --git a/dev/test/metabase/core/module_history_test.clj b/dev/test/metabase/core/module_history_test.cljindex 47fa95bc064..8930d28d53a 100644--- a/dev/test/metabase/core/module_history_test.clj+++ b/dev/test/metabase/core/module_history_test.clj@@ -6,7 +6,8 @@[dev.deps-graph :as deps-graph][dev.module-history :as module-history][dev.module-source :as module-source]- [hooks.common.modules]))+ [hooks.common.modules]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -291,8 +292,8 @@(deftest commits-to-measure-test(testing "an explicit revision is measured once when the sample also picks it"- (let [head (str/trim (#'module-history/git! ["rev-parse" "HEAD"]))- ;; A date after every commit samples only the tip of `ref`.- tomorrow (str (.plusDays (java.time.LocalDate/now java.time.ZoneOffset/UTC) 1))]- (is (= [head]- (#'module-history/commits-to-measure {:commits ["HEAD"], :ref "HEAD", :since tomorrow}))))))+ ;; Stubbed, because the sample walks git history, which a shallow CI checkout does not have.+ (mt/with-dynamic-fn-redefs [module-history/resolve-commit (fn [commit] (get {"HEAD" "abc"} commit commit))+ module-history/sample-commits (constantly ["abc" "def"])]+ (is (= ["abc" "def"]+ (#'module-history/commits-to-measure {:commits ["HEAD"], :since "2026-01-01"}))))))diff --git a/mage/src/mage/jar_download.clj b/mage/src/mage/jar_download.cljindex bec96f539b4..fd8e7713907 100644--- a/mage/src/mage/jar_download.clj+++ b/mage/src/mage/jar_download.clj@@ -3,8 +3,6 @@[babashka.curl :as curl][babashka.fs :as fs][babashka.process :as p]- ;; mage runs under babashka, which bundles cheshire; metabase.util.json isn't on its classpath- ^{:clj-kondo/ignore [:discouraged-namespace]}[cheshire.core :as json][clojure.edn :as edn][clojure.java.io :as io]diff --git a/mage/src/mage/module_history.clj b/mage/src/mage/module_history.cljnew file mode 100644index 00000000000..7645ccde5eb--- /dev/null+++ b/mage/src/mage/module_history.clj@@ -0,0 +1,332 @@+(ns mage.module-history+ "Keeps the files that track how the backend module graph changed up to date, and publishes them.++ The measuring and the files are `dev.module-history-report`'s work. Publishing is described by an EDN file kept+ outside the repo, since it names one person's or one team's uploads, cards and pages:++ {:pages [{:template \"explorer-template.html\", :file \"module-boundaries.html\"}]+ :metabase {:profile \"stats\"+ :uploads {\"history.csv\" 123, \"structure.csv\" 124, \"cycles.csv\" 125}+ :cards [456 457]}+ :shares {\"module-boundaries.html\" \"module-boundaries-a1b2c3\"}}++ - `:pages` fills each `:template`, a path relative to the EDN file, from `history.json` and writes it to `:file`+ in the output directory. See [[fill-template]] for the placeholders.+ - `:metabase` replaces the rows of each upload table with a file from the output directory, using the `mb` CLI+ and its `:profile`. A replace needs the same columns as the upload. Metabase caches a card's results by its+ query, so each of the native `:cards` then gets a new comment at the end of its SQL.+ - `:shares` overwrites each mb-shares slug with a file from the output directory, using the `share` CLI."+ (:require+ [babashka.fs :as fs]+ [cheshire.core :as json]+ [clojure.edn :as edn]+ [clojure.string :as str]+ [mage.color :as c]+ [mage.shell :as shell]+ [mage.util :as u])+ (:import+ (java.time Instant)+ (java.time.temporal ChronoUnit)+ (java.util.regex Pattern)))++(set! *warn-on-reflection* true)++;;;; Running commands++(defn- sh!+ "Run `args` and return its stdout lines, exiting when the command fails."+ [opts args]+ (let [{:keys [exit out err]} (apply shell/sh* opts args)]+ (when-not (zero? exit)+ (u/exit (c/red (str "Failed: " (str/join " " args)+ (when (seq err) (str "\n" (str/join "\n" err)))))+ exit))+ out))++(defn- change!+ "Run `args`, which changes something outside this checkout. With `dry-run?`, print the command instead."+ [dry-run? args]+ (if dry-run?+ (println (c/yellow "Would run:") (str/join " " args))+ (sh! {} args)))++;;;; Measuring++(def ^:private measuring-timeout-ms+ "Enough to measure the whole sample with an empty cache."+ (* 3 60 60 1000))++(defn- measure!+ "Write the files into `dir`, for the history of `ref` when it is given and of `origin/master` otherwise."+ [dir ref]+ ;; Always a fresh JVM, never the dev REPL: results are cached under a hash of the measuring code on disk, and a+ ;; REPL may have loaded an older version of it.+ (sh! {:timeout-ms measuring-timeout-ms}+ (cond-> ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" (pr-str (str dir))]+ ref (conj ":ref" (pr-str ref)))))++;;;; Publishing config++(defn- unknown-keys+ "A row for [[config-checks]]: the map at `path`, or each map in the sequence there, holds only `known` keys.+ Any other value passes, and is left to the row that checks its shape."+ [path known]+ (let [unknown (fn [value]+ (let [maps (cond (map? value) [value], (sequential? value) (filter map? value))]+ (distinct (mapcat #(remove known (keys %)) maps))))]+ [path+ (comp empty? unknown)+ #(str "has unknown keys: " (str/join ", " (unknown %)))]))++(defn- map-from-strings-to+ "A predicate for a map from strings to values that pass `valid-value?`."+ [valid-value?]+ (fn [m] (and (map? m) (every? string? (keys m)) (every? valid-value? (vals m)))))++(defn- page? [page]+ (and (map? page) (string? (:template page)) (string? (:file page))))++(def ^:private config-checks+ "What a publishing config must satisfy, as `[path valid? message]` in the order problems are reported.+ A check is skipped when nothing is at `path`.+ A `message` is a string, or a function of the value there."+ [(unknown-keys [] #{:metabase :pages :shares})+ (unknown-keys [:pages] #{:file :template})+ [[:pages]+ #(and (sequential? %) (every? page? %))+ "must be a vector of {:template \"path\", :file \"name\"}"]+ (unknown-keys [:metabase] #{:cards :profile :uploads})+ [[:metabase]+ #(string? (:profile %))+ "needs :profile, the mb profile to use"]+ [[:metabase :uploads]+ (map-from-strings-to pos-int?)+ "must map a file name to an upload table id"]+ [[:metabase :cards]+ #(and (sequential? %) (every? pos-int? %))+ "must be a vector of card ids"]+ [[:shares]+ (map-from-strings-to string?)+ "must map a file name to an mb-shares slug"]])++(defn- config-problems+ "Everything wrong with publishing `config`, as `{:path [:metabase :cards], :message \"must be ...\"}` maps.+ The path is empty for the config itself. Empty when the config is usable."+ [config]+ (if-not (map? config)+ [{:path [], :message "must hold one map"}]+ (for [[path valid? message] config-checks+ :let [value (get-in config path)]+ :when (and (some? value) (not (valid? value)))]+ {:path path+ :message (if (fn? message) (message value) message)})))++(defn- problem-line+ [{:keys [path message]}]+ (str (if (seq path) (str/join " " path) "the config") " " message))++(defn- read-config+ "The publishing config in `file`. Exits when the file is missing or malformed."+ [file]+ (when-not (fs/exists? file)+ (u/exit (c/red (str "No publishing config at " file)) 1))+ (let [config (try+ (edn/read-string (slurp (str file)))+ (catch Exception e+ (u/exit (c/red (str file " is not readable EDN: " (ex-message e))) 1)))+ problems (config-problems config)]+ (when (seq problems)+ (u/exit (c/red (str file " is not a publishing config:\n " (str/join "\n " (map problem-line problems)))) 1))+ config))++(defn- missing-tools+ "The command-line tools that publishing `config` runs and that are not installed."+ [{:keys [metabase shares]}]+ (binding [u/*skip-warning* true]+ (into [] (remove u/can-run?) (cond-> []+ (or (seq (:uploads metabase)) (seq (:cards metabase))) (conj "mb")+ (seq shares) (conj "share")))))++;;;; Pages++(defn- script-safe-json+ "`value` as JSON that can sit inside a `<script>` element. Every `<` is escaped, since `</script>` ends the element+ and `<!--` can make the parser skip the tag that would end it. So are the line and paragraph separators, which+ older JavaScript parsers do not accept inside a string."+ [value]+ (str/replace (json/generate-string value) #"[<\u2028\u2029]" #(format "\\u%04x" (int (first %)))))++(def ^:private placeholders+ "Placeholder in a page template -> the key of `history.json` it shows, and how to write that key's value."+ [["/*DATA*/" "commits" script-safe-json]+ ["/*TODAY*/" "today" script-safe-json]+ ["/*ANCHORS*/" "clusters" script-safe-json]+ ["/*RETEST*/" "retest" script-safe-json]+ ["/*PAIRS*/" "pairs" script-safe-json]+ ["/*CUTS*/" "cuts" script-safe-json]+ ["/*COUNT*/" "commits" count]])++(defn fill-template+ "The page `template` with its placeholders replaced from `page-data`, the contents of `history.json`:++ - `/*DATA*/`: the commits, a JSON array+ - `/*TODAY*/`: the cycles at the latest commit, a JSON array+ - `/*ANCHORS*/`: the cluster names, a JSON array+ - `/*RETEST*/`: the latest commit's namespaces per tenth of retest share, a JSON object+ - `/*PAIRS*/`: the module pairs whose upward requires close a cycle at the latest commit, a JSON array+ - `/*CUTS*/`: the cheapest ways to split the largest cycle at the latest commit, a JSON array+ - `/*COUNT*/`: how many commits there are"+ [template page-data]+ ;; One pass over the template, so a placeholder inside a value already written, say in a commit subject, stays.+ (let [by-placeholder (into {} (map (juxt first rest)) placeholders)]+ (str/replace template+ (re-pattern (str/join "|" (map (comp #(Pattern/quote %) first) placeholders)))+ (fn [placeholder]+ (let [[k write] (by-placeholder placeholder)]+ (str (write (get page-data k))))))))++(defn- missing-page-keys+ "Keys that a placeholder in `template` shows and that `page-data` lacks."+ [template page-data]+ (distinct (for [[placeholder k] placeholders+ :when (and (str/includes? template placeholder) (not (contains? page-data k)))]+ k)))++(defn- read-template+ "The page template at `path`. Exits when there is none, or when it shows a key that `page-data` lacks."+ [path page-data]+ (when-not (fs/exists? path)+ (u/exit (c/red (str "No page template at " path)) 1))+ (let [template (slurp (str path))+ missing (missing-page-keys template page-data)]+ (when (seq missing)+ (u/exit (c/red (str "history.json has no " (str/join ", " (map pr-str missing)) ", which " path+ " shows. Run without --no-measure to write it again."))+ 1))+ template))++(defn- write-pages!+ "Fill each of `pages` and write it into `dir`. A template path is relative to `config-dir`."+ [dir config-dir pages]+ (when (seq pages)+ (let [page-data (json/parse-string (slurp (str (fs/path dir "history.json"))))+ ;; Every template is read and checked before any page is written.+ templates (mapv #(read-template (fs/path config-dir (:template %)) page-data) pages)]+ (doseq [[{:keys [file]} template] (map vector pages templates)]+ (spit (str (fs/path dir file)) (fill-template template page-data))+ (println (c/green "Wrote") (str (fs/path dir file)))))))++;;;; Metabase++(def ^:private refresh-comment-prefix "-- refreshed by mage module-history at ")++(defn refreshed-sql+ "Native `sql` ending in one comment that carries `stamp`, replacing the comment an earlier refresh left."+ [sql stamp]+ (let [lines (->> (str/split-lines sql)+ (remove #(str/starts-with? % refresh-comment-prefix))+ (str/join "\n"))]+ (str (str/trimr lines) "\n" refresh-comment-prefix stamp)))++(defn- mb+ "An `mb` command line that asks `profile` for JSON without a size cap."+ [profile args]+ (concat ["mb"] args ["--profile" profile "--json" "--max-bytes" "0"]))++(defn- native-query+ "The dataset query of native card `id`. Exits when the card is not native, or when `profile` may not change it."+ [profile id]+ ;; String keys, so that names inside the query, such as template tags, are sent back as they came.+ (let [card (json/parse-string (str/join "\n" (sh! {:quiet? true} (mb profile ["card" "get" (str id) "--full"]))))+ query (get card "dataset_query")]+ (when-not (string? (get-in query ["stages" 0 "native"]))+ (u/exit (c/red (str "Card " id " is not a native query, so its results cannot be refreshed this way")) 1))+ (when (false? (get card "can_write"))+ (u/exit (c/red (str "Card " id " cannot be changed with the " profile " profile")) 1))+ query))++(defn- refresh-card!+ "Change the SQL of native card `id` so Metabase stops serving the results it cached."+ [dry-run? profile stamp id]+ (if dry-run?+ (println (c/yellow "Would refresh card") id)+ ;; Fetch the query again: an edit made while the uploads ran must not be overwritten with an older copy.+ (let [query (native-query profile id)+ tmp (fs/create-temp-dir {:prefix "module-history"})+ body (str (fs/path tmp "card.json"))]+ (try+ (spit body (json/generate-string+ {"dataset_query" (update-in query ["stages" 0 "native"] refreshed-sql stamp)}))+ (sh! {:quiet? true} (mb profile ["card" "update" (str id) "--file" body]))+ (finally+ (fs/delete-tree tmp)))+ (println (c/green "Refreshed card") id))))++(defn- publish-to-metabase!+ "Replace the upload tables, then refresh the cards. Every card is fetched and checked first, a dry run's too, so a+ card that cannot be refreshed stops the run before any table changes."+ [dry-run? dir {:keys [profile uploads cards]}]+ (run! #(native-query profile %) cards)+ (let [stamp (str (.truncatedTo (Instant/now) ChronoUnit/SECONDS))]+ (doseq [[file table-id] (sort uploads)]+ (change! dry-run? (mb profile ["upload" "replace" (str table-id) "--file" (str (fs/path dir file))])))+ (doseq [id cards]+ (refresh-card! dry-run? profile stamp id))))++;;;; Publishing++(defn- missing-files+ "Files the config publishes that are not in `dir` and that no page will write there."+ [dir {:keys [pages metabase shares]}]+ (->> (concat (keys (:uploads metabase)) (keys shares))+ (remove (set (map :file pages)))+ (remove #(fs/exists? (fs/path dir %)))+ distinct))++(defn- publish!+ "Publish the files in `dir` as `config` says, with page templates relative to `config-dir`. With `dry-run?`, write+ the pages but only print the commands that would change Metabase or a share."+ [dry-run? dir config config-dir]+ (let [missing (missing-files dir config)]+ (when (seq missing)+ (u/exit (c/red (str "Not in " dir ": " (str/join ", " missing))) 1))+ (write-pages! dir config-dir (:pages config))+ (when-let [metabase (:metabase config)]+ (publish-to-metabase! dry-run? dir metabase))+ (doseq [[file slug] (sort (:shares config))]+ (change! dry-run? ["share" "-e" (str (fs/path dir file)) slug]))))++;;;; Entry point++(defn- remote-branch+ "The branch on `origin` that `ref` tracks, or nil for a local branch or a commit, which no fetch changes."+ [ref]+ (second (re-matches #"(?:refs/remotes/)?origin/(.+)" ref)))++(defn cli-update+ "Fetch the ref to measure, measure its new commits, write the files, and publish them when asked."+ [{:keys [options]}]+ (let [{:keys [dir publish ref no-fetch no-measure dry-run]} options]+ (when (str/blank? dir)+ (u/exit (c/red "Pass --dir DIR, the directory to keep the files in.") 1))+ (when (and dry-run (not publish))+ (u/exit (c/red "--dry-run only applies to --publish.") 1))+ (let [dir (fs/absolutize (fs/expand-home dir))+ config-file (some-> publish fs/expand-home)+ ;; Read the config first, so a mistake in it shows before the measuring, not after.+ config (some-> config-file read-config)+ timer (u/start-timer)]+ ;; A dry run checks too, so that it predicts what a real run does.+ (when-let [missing (seq (some-> config missing-tools))]+ (u/exit (c/red (str "Not installed, but needed to publish: " (str/join ", " missing))) 1))+ (when-not no-measure+ (when-not no-fetch+ (when-let [branch (remote-branch (or ref "origin/master"))]+ (println (str "Fetching origin/" branch "..."))+ ;; Name the destination: a clone that fetches only some branches would update only FETCH_HEAD.+ (sh! {} ["git" "fetch" "--quiet" "origin" (str "+refs/heads/" branch ":refs/remotes/origin/" branch)])))+ (measure! dir ref))+ (when config+ (publish! (boolean dry-run) dir config (fs/parent (fs/absolutize config-file))))+ (printf "\nFinished in %.1fs\n" (/ (u/since-ms timer) 1000.0))+ (flush))))diff --git a/mage/src/mage/project_tests.clj b/mage/src/mage/project_tests.cljindex 10292f91a6e..812e6bbf850 100644--- a/mage/src/mage/project_tests.clj+++ b/mage/src/mage/project_tests.clj@@ -9,7 +9,10 @@(def ^:private module-check-namespaces'[dev.modules-config-test+ metabase.core.module-cuts-testmetabase.core.module-cycles-test+ metabase.core.module-history-report-test+ metabase.core.module-history-testmetabase.core.modules-test])(def ^:private honey-sql-check-namespacesdiff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 87e1804a0e8..ffc61201b38 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -23,6 +23,7 @@[mage.kondo-ratchets-history-test][mage.merge-kondo-ratchets-test][mage.merge-yaml-migrations-test]+ [mage.module-history-test][mage.modules-test][mage.new-migration-test][mage.openapi-diff-test]diff --git a/mage/test/mage/module_history_test.clj b/mage/test/mage/module_history_test.cljnew file mode 100644index 00000000000..cd60f1aa6b6--- /dev/null+++ b/mage/test/mage/module_history_test.clj@@ -0,0 +1,258 @@+(ns mage.module-history-test+ (:require+ [babashka.fs :as fs]+ [clojure.string :as str]+ [clojure.test :refer [are deftest is testing]]+ [mage.module-history :as module-history]+ [mage.shell :as shell]+ [mage.util :as u]))++;; Referenced by core_test.clj to ensure namespace is loaded+(def keep-me :loaded)++(set! *warn-on-reflection* true)++(deftest refreshed-sql-test+ (testing "the stamp goes in a comment after the query"+ (is (= "select 1\n-- refreshed by mage module-history at 2026-10-10T00:00:00Z"+ (module-history/refreshed-sql "select 1\n\n\n" "2026-10-10T00:00:00Z"))))+ (testing "a second refresh replaces the first one's comment, and other comments stay"+ (is (= "select 1\n-- by seq\norder by seq\n-- refreshed by mage module-history at 2026-10-11T00:00:00Z"+ (-> "select 1\n-- by seq\norder by seq"+ (module-history/refreshed-sql "2026-10-10T00:00:00Z")+ (module-history/refreshed-sql "2026-10-11T00:00:00Z"))))))++(deftest fill-template-test+ (is (= (str "<p>4 commits</p><script>"+ "const DATA = [{\"subject\":\"Close \\u003c/script> early\"},"+ "{\"subject\":\"Hide \\u003c!--\\u003cscript> \\u2028\"},"+ "{\"subject\":\"$1 \\\\ kept\"},"+ "{\"subject\":\"Rename /*COUNT*/ and /*DATA*/\"}];"+ "const TODAY = [{\"cluster\":\"tardis\"}];"+ "const ANCHORS = [\"tardis\"];"+ "const RETEST = {\"module\":[1,2]};"+ "const PAIRS = [{\"module\":\"x.y\"}];"+ "const CUTS = [];"+ "</script>")+ (module-history/fill-template+ (str "<p>/*COUNT*/ commits</p><script>"+ "const DATA = /*DATA*/;const TODAY = /*TODAY*/;"+ "const ANCHORS = /*ANCHORS*/;const RETEST = /*RETEST*/;"+ "const PAIRS = /*PAIRS*/;const CUTS = /*CUTS*/;</script>")+ {"commits" [{"subject" "Close </script> early"}+ {"subject" "Hide <!--<script> \u2028"}+ {"subject" "$1 \\ kept"}+ ;; A placeholder in a value stays as it is.+ {"subject" "Rename /*COUNT*/ and /*DATA*/"}]+ "today" [{"cluster" "tardis"}]+ "clusters" ["tardis"]+ "retest" {"module" [1 2]}+ "pairs" [{"module" "x.y"}]+ "cuts" []}))))++(deftest config-problems-test+ (testing "a full config and an empty one are both usable"+ (is (= []+ (#'module-history/config-problems+ {:pages [{:template "t.html", :file "page.html"}]+ :metabase {:profile "stats", :uploads {"history.csv" 1}, :cards [2 3]}+ :shares {"page.html" "slug-a1"}})))+ (is (= []+ (#'module-history/config-problems {}))))+ (testing "each mistake is reported at the key it is under, a key's problems together"+ (is (= [[] [:pages] [:pages] [:metabase] [:metabase] [:metabase :uploads] [:metabase :cards]]+ (map :path (#'module-history/config-problems+ {:share {}+ :pages [{:template "t.html", :path "page.html"}]+ :metabase {:uploads {"history.csv" "1"}, :cards [:a], :card 2}})))))+ (testing "a value of the wrong kind is a problem at its key, whatever it is"+ (are [config path] (= [path] (distinct (map :path (#'module-history/config-problems config))))+ {:metabase 5} [:metabase]+ {:metabase "stats"} [:metabase]+ {:metabase {:profile "p", :uploads {:a.csv 1}}} [:metabase :uploads]+ {:pages 5} [:pages]+ {:pages {:file "a"}} [:pages]+ {:shares {"a" 1}} [:shares]+ {:shares {:a "slug"}} [:shares]+ {:shares false} [:shares]))+ (testing "a config that is not a map is one problem"+ (is (= [{:path [], :message "must hold one map"}]+ (#'module-history/config-problems []))))+ (testing "a problem reads as its path and what is wrong there"+ (is (= ["the config has unknown keys: :share"+ ":pages has unknown keys: :path"+ ":pages must be a vector of {:template \"path\", :file \"name\"}"+ ":metabase has unknown keys: :card"+ ":metabase :cards must be a vector of card ids"]+ (map #'module-history/problem-line+ (#'module-history/config-problems {:share {}+ :pages [{:template "t.html", :path "p.html"}]+ :metabase {:profile "p", :cards [:a], :card 1}}))))))++(def ^:private native-card+ "What `mb card get` returns for a native card the profile may change."+ "{\"id\": 21, \"can_write\": true, \"dataset_query\": {\"database\": 1, \"stages\": [{\"native\": \"select 1\"}]}}")++(defn- fake-sh+ "A [[mage.shell/sh*]] stand-in that records each command in `calls` and answers `mb card get` with the JSON `card`.+ A command that starts with a key of `failures` exits with that key's value."+ [calls failures card]+ (fn [& args]+ (let [args (vec (if (map? (first args)) (rest args) args))]+ (swap! calls conj (cond-> args+ ;; The update's body is in a temporary file: record what it holds instead of its path.+ (= ["mb" "card" "update"] (take 3 args))+ (update 5 slurp)))+ {:exit (or (some (fn [[start exit]] (when (= start (take (count start) args)) exit)) failures) 0)+ :err []+ :out (if (= ["mb" "card" "get"] (take 3 args))+ [card]+ [])})))++(defn- publish!+ "Publish a directory holding `files` with `config`, on a machine where `tools` are installed and `failures` are the+ commands that fail. Returns the commands that ran with the directory's path replaced by `DIR` and the time by+ `STAMP`, the page it wrote, its exit code, and what it printed."+ [{:keys [config files options tools failures card]}]+ (let [dir (fs/create-temp-dir {:prefix "module-history-test"})]+ (doseq [[file-name content] (merge {"publish.edn" (pr-str config), "template.html" "/*COUNT*/ of /*ANCHORS*/"}+ files)]+ (spit (str (fs/path dir file-name)) content))+ (let [calls (atom [])+ exit (atom 0)+ printed (with-out-str+ (with-redefs [shell/sh* (fake-sh calls failures card)+ u/can-run? (fn [tool] (contains? tools tool))]+ (try+ (module-history/cli-update+ {:options (merge {:dir (str dir)+ :publish (str (fs/path dir "publish.edn"))+ :no-measure true}+ options)})+ (catch clojure.lang.ExceptionInfo e+ (reset! exit (:babashka/exit (ex-data e)))))))]+ (try+ {:exit @exit+ :calls (mapv (fn [call]+ (mapv #(-> (str %)+ (str/replace (str dir) "DIR")+ (str/replace #"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ" "STAMP"))+ call))+ @calls)+ :page (when (fs/exists? (fs/path dir "page.html")) (slurp (str (fs/path dir "page.html"))))+ :printed printed}+ (finally+ (fs/delete-tree dir))))))++(def ^:private run+ "A publishing run that works, for [[publish!]]."+ {:config {:pages [{:template "template.html", :file "page.html"}]+ :metabase {:profile "stats", :uploads {"history.csv" 11, "cycles.csv" 12}, :cards [21]}+ :shares {"page.html" "slug-a1"}}+ ;; The template shows no retest distribution, so a `history.json` without one is enough.+ :files {"history.csv" "sha\n"+ "cycles.csv" "sha\n"+ "history.json" "{\"commits\": [{}, {}], \"today\": [], \"clusters\": [\"tardis\"]}"}+ :options {}+ :tools #{"mb" "share"}+ :failures {}+ :card native-card})++(defn- naming+ "The `result` of [[publish!]] with what it printed cut down to the first match of `pattern`."+ [pattern result]+ (update result :printed #(re-find pattern %)))++(deftest measure-test+ (testing "measuring runs in a fresh JVM after fetching the ref it measures, when that ref is on origin"+ (let [calls (atom [])]+ (with-out-str+ (with-redefs [shell/sh* (fake-sh calls {} native-card)]+ (module-history/cli-update {:options {:dir "/tmp/out", :no-fetch true}})+ (module-history/cli-update {:options {:dir "/tmp/out"}})+ (module-history/cli-update {:options {:dir "/tmp/out", :ref "origin/stack-tip"}})+ (module-history/cli-update {:options {:dir "/tmp/out", :ref "stack-tip"}})))+ (is (= [["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""]+ ["git" "fetch" "--quiet" "origin" "+refs/heads/master:refs/remotes/origin/master"]+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""]+ ["git" "fetch" "--quiet" "origin" "+refs/heads/stack-tip:refs/remotes/origin/stack-tip"]+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""+ ":ref" "\"origin/stack-tip\""]+ ["clojure" "-X:dev" "dev.module-history-report/update!" ":dir" "\"/tmp/out\""+ ":ref" "\"stack-tip\""]]+ @calls)))))++(def ^:private card-get+ "The command that fetches card 21."+ ["mb" "card" "get" "21" "--full" "--profile" "stats" "--json" "--max-bytes" "0"])++(deftest publish-test+ (testing "pages written, cards checked, uploads replaced, cards fetched again and refreshed, shares updated"+ (is (= {:exit 0+ :page "2 of [\"tardis\"]"+ :calls [card-get+ ["mb" "upload" "replace" "12" "--file" "DIR/cycles.csv" "--profile" "stats" "--json"+ "--max-bytes" "0"]+ ["mb" "upload" "replace" "11" "--file" "DIR/history.csv" "--profile" "stats" "--json"+ "--max-bytes" "0"]+ ;; Fetched again, so an edit made during the uploads is kept.+ card-get+ ;; Only the query is sent, with the rest of it unchanged.+ ["mb" "card" "update" "21" "--file"+ (str "{\"dataset_query\":{\"database\":1,\"stages\":"+ "[{\"native\":\"select 1\\n-- refreshed by mage module-history at STAMP\"}]}}")+ "--profile" "stats" "--json" "--max-bytes" "0"]+ ["share" "-e" "DIR/page.html" "slug-a1"]]}+ (dissoc (publish! run) :printed)))))++(deftest publish-dry-run-test+ (testing "a dry run writes the page, only fetches the cards, and prints what it would run"+ (let [{:keys [printed], :as result} (publish! (assoc run :options {:dry-run true}))]+ (is (= {:exit 0, :page "2 of [\"tardis\"]", :calls [card-get]}+ (dissoc result :printed)))+ (is (= ["mb upload replace 12" "mb upload replace 11" "Would refresh card" "share -e"]+ (re-seq #"mb upload replace \d+|Would refresh card|share -e" printed))))))++(deftest publish-failing-command-test+ (testing "a command that fails stops the run with its exit code, and nothing runs after it"+ (is (= {:exit 3+ :calls [card-get+ ["mb" "upload" "replace" "12" "--file" "DIR/cycles.csv" "--profile" "stats" "--json"+ "--max-bytes" "0"]+ ["mb" "upload" "replace" "11" "--file" "DIR/history.csv" "--profile" "stats" "--json"+ "--max-bytes" "0"]]}+ (select-keys (publish! (assoc run :failures {["mb" "upload" "replace" "11"] 3})) [:exit :calls]))))+ (testing "a card that cannot be fetched stops the run before any upload changes"+ (is (= {:exit 4, :calls [card-get]}+ (select-keys (publish! (assoc run :failures {["mb" "card" "get"] 4})) [:exit :calls]))))+ (testing "a card the profile may not change stops the run before any upload changes"+ (is (= {:exit 1, :calls [card-get]}+ (select-keys (publish! (assoc run :card (str/replace native-card "true" "false"))) [:exit :calls])))))++(deftest publish-missing-file-test+ (testing "nothing is written or published when a file the config names is not there"+ (is (= {:exit 1, :page nil, :calls [], :printed "cycles.csv"}+ (naming #"cycles\.csv" (publish! (update run :files dissoc "cycles.csv")))))))++(deftest publish-bad-config-test+ (testing "a config with a mistake stops the run before anything else happens"+ (is (= {:exit 1, :page nil, :calls [], :printed ":share"}+ (naming #":share\b" (publish! (assoc-in run [:config :share] {})))))))++(deftest publish-missing-tool-test+ (testing "a tool the config needs and that is not installed stops the run, a dry run too"+ (is (= {:exit 1, :page nil, :calls [], :printed "share"}+ (naming #"share" (publish! (assoc run :tools #{"mb"}, :options {:dry-run true}))))))+ (testing "a tool the config would not run need not be installed"+ (is (= {:exit 0, :page "2 of [\"tardis\"]", :calls []}+ (dissoc (publish! (assoc run+ :config {:pages (get-in run [:config :pages])+ :metabase {:profile "stats"}}+ :tools #{}))+ :printed)))))++(deftest publish-stale-history-test+ (testing "no page is written from a history.json that lacks a key its template shows"+ (is (= {:exit 1, :page nil, :calls [], :printed "clusters"}+ (naming #"clusters"+ (publish! (assoc-in run [:files "history.json"] "{\"commits\": [{}, {}], \"today\": []}")))))))diff --git a/mage/test/mage/project_tests_test.clj b/mage/test/mage/project_tests_test.cljindex dc55f7f9c9a..b64b19b54d8 100644--- a/mage/test/mage/project_tests_test.clj+++ b/mage/test/mage/project_tests_test.clj@@ -44,7 +44,10 @@(is (= [["clojure" "-M:test"]["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"(str '[dev.modules-config-test+ metabase.core.module-cuts-testmetabase.core.module-cycles-test+ metabase.core.module-history-report-test+ metabase.core.module-history-testmetabase.core.modules-testmetabase.core.kondo-ratchet-testmetabase.core.kondo-ratchet-check-test@@ -93,7 +96,12 @@(deftest targeted-suites-test(testing "modules runs only its own namespaces"(is (= [["clojure" "-X:dev:dev/test:ee:ee-dev:drivers:drivers-dev:test:ci" ":only"- "[dev.modules-config-test metabase.core.module-cycles-test metabase.core.modules-test]"]]+ (str '[dev.modules-config-test+ metabase.core.module-cuts-test+ metabase.core.module-cycles-test+ metabase.core.module-history-report-test+ metabase.core.module-history-test+ metabase.core.modules-test])]](:calls (run-suites! [] ["modules"])))))(testing "ratchets checks the repository's policy file"(is (= [["./bin/mage" "kondo-ratchets"]]