The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
AppDb support for Data Complexity Score CLI (#74142)
Chris Truter6d2d3af70b85
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.cljindex 139152f9d6d..a891d1f2e2a 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj@@ -4,7 +4,6 @@[metabase-enterprise.data-complexity-score.complexity :as complexity][metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope][metabase-enterprise.data-complexity-score.models.data-complexity-score :as data-complexity-score]- [metabase-enterprise.data-complexity-score.settings :as settings][metabase-enterprise.data-complexity-score.synonym-source :as synonym-source][metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score][metabase.api.common :as api]@@ -86,13 +85,8 @@result (complexity/complexity-scores(assoc (synonym-source/complexity-scores-opts):metabot-scope (metabot-scope/internal-metabot-scope)))- stored (data-complexity-score/record-score! fingerprint result)]- ;; Advance the last-published fingerprint iff Snowplow actually accepted the event — mirrors- ;; the scheduled path's gate in `task.complexity-score/run-scoring!`. Without this, a- ;; superuser-triggered recalculation leaves the setting stale and the next boot would- ;; redundantly re-score even though a valid snapshot was just persisted.- (when (::complexity/snowplow-published? (meta result))- (settings/data-complexity-scoring-last-fingerprint! fingerprint))+ stored (data-complexity-score/record-score! fingerprint "appdb" result)]+ (task.complexity-score/maybe-advance-last-fingerprint! fingerprint result)(m.util/deep-snake-keys (or stored result)))(finally(.set api-scoring-running? false))))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.cljindex cbc028dfaa5..741a59e2f41 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj@@ -1,20 +1,43 @@(ns metabase-enterprise.data-complexity-score.cli- "CLI entrypoint for computing the data complexity score against on-disk representations.+ "CLI entrypoint for computing the data complexity score.- Useful for offline scoring of benchmarks, CI, and exploratory analysis.+ Two source modes:- Invoke via:+ `--source representation` (default) — scores an on-disk serdes export. No appdb required+ unless `--write-to-appdb true` is also requested. Useful for offline scoring of+ benchmarks, CI, and exploratory analysis.- clojure -M:ee:dev -m metabase-enterprise.data-complexity-score.cli \\- --representation-dir path/to/fixture/+ `--source appdb` — bootstraps the application database and scores live entities the same+ way the cron job (`task.complexity-score`) and API endpoint do.- Patterned after [[metabase-enterprise.checker.cli]] — same `fail!` pattern, same arg style."+ Persistence is controlled independently by `--write-to-appdb`. The default tracks the source:+ true in appdb mode, false in representation mode. Both can be overridden explicitly.++ Two invocation styles:++ Dev (classpath, no AOT):+ clojure -M:ee:dev -m metabase-enterprise.data-complexity-score.cli --source appdb+ clojure -M:ee:dev -m metabase-enterprise.data-complexity-score.cli \\+ --source representation -r path/to/fixture/++ AOT JAR (via `metabase.core.bootstrap`):+ java -jar metabase.jar --mode complexity-score --source appdb+ java -jar metabase.jar --mode complexity-score \\+ --source representation -r path/to/fixture/++ Patterned after [[metabase-enterprise.checker.cli]] — same `fail!` pattern, same arg style,+ and the same bootstrap-dispatched [[entrypoint]] hook."(:require[clojure.java.io :as io][clojure.pprint :as pprint][clojure.tools.cli :as cli][metabase-enterprise.data-complexity-score.complexity :as complexity]- [metabase-enterprise.data-complexity-score.representation :as representation]))+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase-enterprise.data-complexity-score.models.data-complexity-score :as data-complexity-score]+ [metabase-enterprise.data-complexity-score.representation :as representation]+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]+ [metabase.app-db.core :as mdb]))(set! *warn-on-reflection* true)@@ -31,6 +54,14 @@(flush)(System/exit 1))+(defn- fail-with-trace!+ "Print `t`'s stack trace (including `caused by` chain) to stderr, then fail! with a one-line summary.+ Used for unexpected throwables — keeps the SQL exception / column-mismatch site visible instead of+ collapsing it into just `<class>: <message>`."+ [^Throwable t]+ (.printStackTrace t)+ (fail! (format "%s: %s" (.getName (class t)) (.getMessage t))))+(defn- validate-dir! [path](let [f (io/file path)](cond@@ -39,14 +70,30 @@(not (.isDirectory f)) (throw (ex-info (str "--representation-dir must be a directory: " path){:cli-validation true :path path})))))+(def ^:private unset+ "Sentinel for `--write-to-appdb` so we can tell 'user didn't pass it' apart from an explicit false."+ ::unset)+(def ^:private cli-options- [["-r" "--representation-dir PATH" "Directory of a serdes YAML export, optionally with an embeddings.json sidecar (required)."]+ [["-s" "--source SOURCE" "'representation' (default) or 'appdb'."+ :default "representation"+ :validate [#{"representation" "appdb"} "--source must be 'representation' or 'appdb'."]]+ ["-r" "--representation-dir PATH" "Directory of a serdes YAML export. Required when --source is 'representation'."]["-e" "--embeddings PATH" "Explicit embeddings JSON file. Overrides embeddings.json in the directory."]+ ["-w" "--write-to-appdb BOOL" "Persist the score row. Defaults: true when --source is 'appdb', false otherwise."+ :default unset+ :parse-fn #(case % "true" true "false" false ::invalid)+ :validate [#(not= % ::invalid) "--write-to-appdb must be 'true' or 'false'."]]["-o" "--output PATH" "Write EDN result to this file instead of stdout."]+ ;; Consumed by `metabase.core.bootstrap` when invoked as `java -jar metabase.jar --mode …`. Declared+ ;; here so tools.cli accepts it and `parse-opts` doesn't report it as an unknown flag.+ [nil "--mode MODE" "(JAR invocation only; handled by bootstrap before reaching this CLI.)"]["-h" "--help" "Show this message."]])(defn- usage [summary]- (str "Usage: clojure -M:ee:dev -m metabase-enterprise.data-complexity-score.cli [options]\n\n"+ (str "Usage:\n"+ " clojure -M:ee:dev -m metabase-enterprise.data-complexity-score.cli [options]\n"+ " java -jar metabase.jar --mode complexity-score [options]\n\n""Options:\n" summary))(defn- pretty [result]@@ -62,24 +109,81 @@(print (pretty result))(flush))))+(defn- resolve-write?+ "Apply the source-driven default for `--write-to-appdb` when it wasn't passed explicitly."+ [{:keys [write-to-appdb]} appdb-source?]+ (if (= write-to-appdb unset) appdb-source? write-to-appdb))++(defn- validate-options!+ "Throws `ex-info` with `:cli-validation true` on user-facing misconfigurations."+ [{:keys [source representation-dir embeddings]}]+ (case source+ "appdb"+ (when (or representation-dir embeddings)+ (throw (ex-info "--source appdb does not accept --representation-dir or --embeddings"+ {:cli-validation true})))++ "representation"+ (do+ (when-not representation-dir+ (throw (ex-info "Missing --representation-dir option (required for --source representation)"+ {:cli-validation true})))+ (validate-dir! representation-dir))))++(defn- run-appdb-mode!+ "Score against the live appdb the same way the cron does; optionally persist."+ [write?]+ (mdb/setup-db-without-migrations!)+ (let [result (complexity/complexity-scores+ (assoc (synonym-source/complexity-scores-opts)+ :metabot-scope (metabot-scope/internal-metabot-scope)))]+ (when write?+ (let [fp (task.complexity-score/current-fingerprint)]+ (data-complexity-score/record-score! fp "appdb" result)+ (task.complexity-score/maybe-advance-last-fingerprint! fp result)))+ result))++(defn- run-representation-mode!+ "Score against an on-disk serdes export; optionally persist with `source` = `representation:<digest>`."+ [{:keys [representation-dir embeddings]} write?]+ (when write?+ (mdb/setup-db-without-migrations!))+ (let [{:keys [library universe embedder digest]} (representation/load-dir representation-dir+ :embeddings-path embeddings)+ result (complexity/score-from-entities library universe embedder {})]+ (when write?+ (data-complexity-score/record-score! (task.complexity-score/current-fingerprint)+ (str "representation:" digest)+ result))+ result))++(defn- with-defaults+ "Apply parse-opts-style defaults so library callers (tests, REPL) don't have to thread them."+ [options]+ (cond-> options+ (nil? (:source options)) (assoc :source "representation")+ (nil? (:write-to-appdb options)) (assoc :write-to-appdb unset)))+(defn- run-cli- "Pure core of the CLI: validates options, loads representation, computes the score.+ "Pure core of the CLI: validates options and dispatches to the source-specific runner.Returns the result map so tests can call this directly without intercepting `System/exit`.- Throws `ex-info` for validation failures (with `:cli-validation true` in the ex-data) and propagates- `load-dir`'s `ex-info` (e.g. missing `--embeddings` override). `-main` converts those into user-facing- `fail!`; library callers can inspect the ex-data and render errors their own way."- [{:keys [representation-dir embeddings]}]- (when-not representation-dir- (throw (ex-info "Missing --representation-dir option" {:cli-validation true})))- (validate-dir! representation-dir)- (let [{:keys [library universe embedder]} (representation/load-dir representation-dir :embeddings-path embeddings)]- (complexity/score-from-entities library universe embedder {})))--;; `-main` is invoked via `clj -M:ee:dev -m …cli`, not via an AOT'd jar, so `(:gen-class)` is unnecessary.-#_{:clj-kondo/ignore [:main-without-gen-class]}-(defn -main- "Entrypoint."- [& args]+ Throws `ex-info` for validation failures (`:cli-validation true`) and propagates `representation/load-dir`'s+ `ex-info` (e.g. missing `--embeddings` override).+ [[entrypoint]] converts those to `fail!`; library callers can inspect the ex-data themselves."+ [options]+ (let [options (with-defaults options)]+ (validate-options! options)+ (let [appdb-source? (= (:source options) "appdb")+ write? (resolve-write? options appdb-source?)]+ (if appdb-source?+ (run-appdb-mode! write?)+ (run-representation-mode! options write?)))))++(defn entrypoint+ "Main entrypoint. Receives raw args (a seq) and owns the process — it always calls `System/exit`.+ Called from [[metabase.core.bootstrap/run-standalone-mode]] via `--mode complexity-score`+ in the AOT JAR, and delegated to from [[-main]] for the `clj -M:ee:dev -m …` dev path."+ [args](let [{:keys [options errors summary]} (cli/parse-opts args cli-options)](cond(:help options) (do (output! (usage summary)) (System/exit 0))@@ -89,9 +193,23 @@(catch clojure.lang.ExceptionInfo e;; Handle CLI validation failures (`:cli-validation`) and missing-embeddings;; failures from representation (`:embeddings-path`) — both are user-facing- ;; misconfigurations rather than bugs.+ ;; misconfigurations rather than bugs. Everything else gets the stack trace+ ;; so an `ex-cause` SQL exception remains diagnosable from the terminal.(let [data (ex-data e)](if (or (:cli-validation data) (:embeddings-path data))(fail! (ex-message e))- (throw e)))))))- (System/exit 0))+ (fail-with-trace! e))))+ (catch Throwable t+ ;; Most likely an older appdb missing a column/table the scorer reads —+ ;; print the trace so the failing query/site is visible.+ (fail-with-trace! t))))+ (System/exit 0)))++;; No `(:gen-class)` — the AOT JAR routes through [[metabase.core.bootstrap]] and dispatches to+;; [[entrypoint]] via `requiring-resolve`, so a class-file `-main` would be dead weight. `-main`+;; only exists for the `clj -M:ee:dev -m …cli` dev path.+#_{:clj-kondo/ignore [:main-without-gen-class]}+(defn -main+ "Dev entrypoint. Delegates to [[entrypoint]], which owns the process and calls `System/exit`."+ [& args]+ (entrypoint args))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.cljindex ae0cf9e2e84..0c1a1fd7541 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj@@ -20,21 +20,25 @@(defn latest-entry"Return the most recently persisted Data Complexity Score row for `fingerprint`, or nil if none exist."- [fingerprint]- (t2/select-one :model/DataComplexityScore :fingerprint fingerprint {:order-by [[:id :desc]]}))+ ([fingerprint] (latest-entry fingerprint "appdb"))+ ([fingerprint source]+ (t2/select-one :model/DataComplexityScore+ :fingerprint fingerprint :source source {:order-by [[:id :desc]]})))(defn latest-score"Return the latest persisted Data Complexity Score payload for `fingerprint`, or nil if none exist."- [fingerprint]- (some-> (latest-entry fingerprint)- score-with-calculated-at))+ ([fingerprint] (latest-score fingerprint "appdb"))+ ([fingerprint source]+ (some-> (latest-entry fingerprint source)+ score-with-calculated-at)))(defn record-score!"Persist one append-only Data Complexity Score snapshot."- [fingerprint score]+ [fingerprint source score](let [id (t2/insert-returning-pk! :model/DataComplexityScore{:fingerprint fingerprint+ :source source:score_data score})](if id(score-with-calculated-at (t2/select-one :model/DataComplexityScore :id id))- (latest-score fingerprint))))+ (latest-score fingerprint source))))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.cljindex 179efd306fc..31ef71bf09c 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj@@ -29,7 +29,10 @@[metabase.util.json :as json][metabase.util.yaml :as yaml])(:import- (java.io File)))+ (java.io File)+ (java.nio.file Files)+ (java.nio.file OpenOption)+ (java.security DigestInputStream MessageDigest)))(set! *warn-on-reflection* true)@@ -201,10 +204,55 @@(let [default (io/file dir "embeddings.json")](if (.exists default) (json/decode (slurp default) false) {}))))+;;; ------------------------------------------- directory digest -------------------------------------------++(defn- sha256-bytes ^bytes [^bytes input]+ (.digest (MessageDigest/getInstance "SHA-256") input))++(defn- hex [^bytes bs]+ (let [sb (StringBuilder. (* 2 (alength bs)))]+ (dotimes [i (alength bs)]+ (.append sb (format "%02x" (bit-and (aget bs i) 0xff))))+ (.toString sb)))++(def ^:private ^"[Ljava.nio.file.OpenOption;" no-open-options+ (make-array OpenOption 0))++(defn- sha256-file-hex+ "SHA-256 of a file's bytes, streamed in 64 KiB chunks to keep large exports off the heap."+ [^File f]+ (let [md (MessageDigest/getInstance "SHA-256")+ buf (byte-array 65536)]+ (with-open [^DigestInputStream is (DigestInputStream. (Files/newInputStream (.toPath f) no-open-options) md)]+ (while (not= -1 (.read is buf))))+ (hex (.digest md))))++(defn dir-digest+ "Stable SHA-256 over the file contents of `dir`.+ Two directories with byte-identical files at the same relative paths produce the same digest.+ Used as the `<digest>` part of the `source` column on `data_complexity_score` rows.++ Relative paths are joined with `/` regardless of the host OS so that a digest computed on+ Linux/macOS matches one computed on Windows for the same export."+ [dir]+ (let [dir-file (io/file dir)+ root (.toPath dir-file)+ entries (->> (file-seq dir-file)+ (filter #(.isFile ^File %))+ (map (fn [^File f]+ [(->> (.relativize root (.toPath f))+ .iterator+ iterator-seq+ (map str)+ (str/join "/"))+ (sha256-file-hex f)]))+ (sort-by first))]+ (hex (sha256-bytes (.getBytes (pr-str entries) "UTF-8")))))+;;; ------------------------------------------- public entry point -------------------------------------------(defn load-dir- "Load a serdes export and return `{:library :universe :embedder}` for the complexity scorer.+ "Load a serdes export and return `{:library :universe :embedder :digest}` for the complexity scorer.Filters mirror the live appdb scorer:- Universe Cards : `type ∈ {metric model}`, not archived@@ -243,4 +291,5 @@(mapv ->table-entity (filter library-table? tables)))):universe (vec (concat (mapv ->card-entity (filter universe-card? cards))(mapv ->table-entity (filter universe-table? tables))))- :embedder (embedders/file-embedder embeddings)}))+ :embedder (embedders/file-embedder embeddings)+ :digest (dir-digest dir-file)}))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.cljindex d8d7ebcf434..6c053062d6e 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj@@ -40,6 +40,15 @@:weights complexity/weights}(synonym-source/fingerprint-fragment)))))+(defn maybe-advance-last-fingerprint!+ "Advance [[settings/data-complexity-scoring-last-fingerprint]] only if Snowplow accepted the publish.+ On failure we leave it untouched so the next boot or cron retries.+ Shared by the cron, API recompute, and CLI appdb paths to keep all persisters in lockstep."+ [fingerprint result]+ (if (::complexity/snowplow-published? (meta result))+ (settings/data-complexity-scoring-last-fingerprint! fingerprint)+ (log/warn "Data Complexity Score: Snowplow publish failed; leaving fingerprint unchanged so the next boot or cron retries")))+(defn- run-scoring!"One scoring pass. Gated by [[settings/data-complexity-scoring-enabled]] so admins can silencescoring without unscheduling the job.@@ -59,10 +68,8 @@(assoc (synonym-source/complexity-scores-opts):metabot-scope (metabot-scope/internal-metabot-scope)))](try- (data-complexity-score/record-score! claim-fingerprint result)- (if (::complexity/snowplow-published? (meta result))- (settings/data-complexity-scoring-last-fingerprint! claim-fingerprint)- (log/warn "Data Complexity Score: Snowplow publish failed; leaving fingerprint unchanged so the next boot or cron retries"))+ (data-complexity-score/record-score! claim-fingerprint "appdb" result)+ (maybe-advance-last-fingerprint! claim-fingerprint result)(catch Throwable t(log/warn t "Data Complexity Score: failed to persist score snapshot; leaving fingerprint unchanged so the next boot or cron retries")))result)diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.cljindex ed8596f1946..f6eadcf0357 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj@@ -92,7 +92,8 @@(reset! captured-fingerprint fingerprint)(with-sample-calculated-at sample-score))](let [resp (mt/user-http-request :crowberto :get 200 endpoint)]- (is (= "api-test-fp" @captured-fingerprint))+ (is (= "api-test-fp" @captured-fingerprint)+ "API passes the cron fingerprint through; source=\"appdb\" defaulting is covered by latest-score-filters-by-source-test")(is (= (m.util/deep-snake-keys (with-sample-calculated-at sample-score)) resp))(is (contains? (:meta resp) :formula_version))(is (= sample-calculated-at (get-in resp [:meta :calculated_at])))@@ -112,12 +113,13 @@(with-redefs [metabot-scope/internal-metabot-scope (constantly {})task.complexity-score/current-fingerprint (constantly "api-test-fp")complexity/complexity-scores (fn [& _] sample-score)- data-complexity-score/record-score! (fn [fingerprint stored-score]- (reset! persisted? [fingerprint stored-score])+ data-complexity-score/record-score! (fn [fingerprint source stored-score]+ (reset! persisted? [fingerprint source stored-score])(with-sample-calculated-at stored-score))](is (= (m.util/deep-snake-keys (with-sample-calculated-at sample-score))(mt/user-http-request :crowberto :get 200 endpoint :force-recalculation true)))- (is (= ["api-test-fp" sample-score] @persisted?))))))+ (is (= ["api-test-fp" "appdb" sample-score] @persisted?)+ "API recompute path must stamp source=\"appdb\"")))))(deftest ^:sequential complexity-endpoint-force-recalculation-advances-last-fingerprint-on-snowplow-publish-test(testing "force recalculation mirrors the scheduled path's fingerprint gate — advance only when Snowplow accepted the event"diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.cljindex 935c8568605..51bc6ffeeff 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj@@ -6,7 +6,12 @@[metabase-enterprise.data-complexity-score.cli :as cli][metabase-enterprise.data-complexity-score.complexity :as complexity][metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase-enterprise.data-complexity-score.models.data-complexity-score :as data-complexity-score][metabase-enterprise.data-complexity-score.representation :as representation]+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]+ [metabase.app-db.core :as mdb][metabase.test :as mt][metabase.util :as u]))@@ -273,3 +278,106 @@(is (empty? extra) "fail! should be called with a single message")(is (re-find #"does-not-exist\.json" msg)"the user-facing message must mention the missing file")))))++;;; ----------------------------------- source + write-to-appdb dispatch -----------------------------------++(deftest ^:parallel run-cli-rejects-appdb-source-combined-with-representation-dir-test+ (testing "--source appdb + --representation-dir is a user error — these flags name mutually exclusive inputs"+ (let [ex (try (#'cli/run-cli {:source "appdb"+ :representation-dir representation-fixture-dir})+ nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? ex))+ (is (true? (:cli-validation (ex-data ex))))+ (is (re-find #"--source appdb does not accept" (ex-message ex)))))+ (testing "--source appdb + --embeddings is also rejected"+ (let [ex (try (#'cli/run-cli {:source "appdb"+ :embeddings "embeddings.json"})+ nil+ (catch clojure.lang.ExceptionInfo e e))]+ (is (some? ex))+ (is (true? (:cli-validation (ex-data ex))))+ (is (re-find #"--source appdb does not accept" (ex-message ex))))))++(deftest ^:sequential run-cli-representation-mode-default-does-not-write-test+ (testing "representation mode with no --write-to-appdb flag never calls record-score! or bootstrap"+ (let [persisted? (atom false)+ bootstrapped? (atom false)]+ (mt/with-dynamic-fn-redefs [data-complexity-score/record-score! (fn [& _] (reset! persisted? true))+ mdb/setup-db-without-migrations! (fn [] (reset! bootstrapped? true))]+ (#'cli/run-cli {:representation-dir representation-fixture-dir})+ (is (false? @persisted?) "representation+no-write must not persist anything")+ (is (false? @bootstrapped?) "representation+no-write must not boot the appdb at all")))))++(deftest ^:sequential run-cli-representation-mode-with-write-stamps-representation-source-test+ (testing "representation + --write-to-appdb true persists a row stamped 'representation:<digest>' and does not advance the cron fingerprint"+ (let [calls (atom [])+ advance-calls (atom 0)]+ (mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])+ task.complexity-score/current-fingerprint (constantly "test-fp")+ task.complexity-score/maybe-advance-last-fingerprint! (fn [& _]+ (swap! advance-calls inc))+ data-complexity-score/record-score! (fn [fp source _result]+ (swap! calls conj [fp source]))]+ (#'cli/run-cli {:representation-dir representation-fixture-dir+ :write-to-appdb true})+ (is (= 1 (count @calls)) "exactly one row written")+ (let [[fp source] (first @calls)]+ (is (= "test-fp" fp))+ (is (re-find #"^representation:[0-9a-f]{64}$" source)+ "source must be 'representation:<sha-256 hex>'"))+ (is (zero? @advance-calls)+ "representation-derived rows must never advance the cron's last-fingerprint setting")))))++(deftest ^:sequential run-cli-appdb-mode-defaults-to-writing-test+ (testing "appdb mode with no --write-to-appdb flag defaults to writing (true)"+ (let [calls (atom [])+ advance-calls (atom [])]+ (mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])+ complexity/complexity-scores (fn [& _] {:meta {}})+ synonym-source/complexity-scores-opts (constantly {})+ metabot-scope/internal-metabot-scope (constantly {})+ task.complexity-score/current-fingerprint (constantly "appdb-fp")+ task.complexity-score/maybe-advance-last-fingerprint! (fn [fp _result]+ (swap! advance-calls conj fp))+ data-complexity-score/record-score! (fn [fp source _result]+ (swap! calls conj [fp source]))]+ (#'cli/run-cli {:source "appdb"})+ (is (= [["appdb-fp" "appdb"]] @calls)+ "appdb-mode default must write one row stamped source=\"appdb\"")+ (is (= ["appdb-fp"] @advance-calls)+ "appdb-mode write path must call maybe-advance-last-fingerprint! the same way the cron does")))))++(deftest ^:sequential run-cli-appdb-mode-respects-explicit-no-write-test+ (testing "appdb + --write-to-appdb false scores but never persists or advances the fingerprint"+ (let [persisted? (atom false)+ advance-calls (atom 0)]+ (mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])+ complexity/complexity-scores (fn [& _] {:meta {}})+ synonym-source/complexity-scores-opts (constantly {})+ metabot-scope/internal-metabot-scope (constantly {})+ data-complexity-score/record-score! (fn [& _] (reset! persisted? true))+ task.complexity-score/maybe-advance-last-fingerprint! (fn [& _]+ (swap! advance-calls inc))]+ (#'cli/run-cli {:source "appdb" :write-to-appdb false})+ (is (false? @persisted?))+ (is (zero? @advance-calls))))))++(deftest ^:parallel dir-digest-is-stable-and-content-sensitive-test+ (testing "dir-digest produces the same value for the same content"+ (let [d1 (#'representation/dir-digest representation-fixture-dir)+ d2 (#'representation/dir-digest representation-fixture-dir)]+ (is (= d1 d2) "two calls against the same dir must produce the same digest")+ (is (re-matches #"[0-9a-f]{64}" d1) "digest must be a 64-char lowercase hex string (SHA-256)")))+ (testing "dir-digest changes when file content changes — guards against a constant-digest regression"+ ;; A regression that made `dir-digest` return a fixed value (e.g. `(hex (sha-256 (pr-str [])))`)+ ;; would pass the stability assertion above. Mutating a single byte in a single file must change+ ;; the digest.+ (let [tmp-dir (empty-tmp-dir "dir-digest-")+ target (io/file tmp-dir "marker.txt")]+ (spit target "original")+ (let [before (#'representation/dir-digest (.getAbsolutePath tmp-dir))]+ (spit target "originalX")+ (let [after (#'representation/dir-digest (.getAbsolutePath tmp-dir))]+ (is (not= before after)+ "appending a byte to a file must change the digest"))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex e16ebc87854..b022e17711d 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -978,15 +978,37 @@fingerprint "latest-score-test/current"](try(t2/delete! :model/DataComplexityScore :fingerprint [:in [other-fingerprint fingerprint]])- (data-complexity-score/record-score! other-fingerprint {:meta {:label "other"}})- (data-complexity-score/record-score! fingerprint {:meta {:label "older"}})- (data-complexity-score/record-score! fingerprint {:meta {:label "newer"}})+ (data-complexity-score/record-score! other-fingerprint "appdb" {:meta {:label "other"}})+ (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "older"}})+ (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "newer"}})(let [score (data-complexity-score/latest-score fingerprint)](is (= "newer" (get-in score [:meta :label])))(is (some? (get-in score [:meta :calculated-at]))))(finally(t2/delete! :model/DataComplexityScore :fingerprint [:in [other-fingerprint fingerprint]]))))))+(deftest ^:sequential latest-score-filters-by-source-test+ (testing "passing source filters out representation-derived rows that share the cron's fingerprint"+ ;; The CLI's representation mode writes under the same `task.complexity-score/current-fingerprint`+ ;; as the cron/API so that an admin re-running the cron after a CLI scoring still benefits from+ ;; the fingerprint short-circuit. Provenance is encoded in `source` ("appdb" vs+ ;; "representation:<digest>"). Read paths that should only surface authoritative rows must pass+ ;; the source filter — otherwise the most-recent representation row would shadow the appdb row.+ (mt/initialize-if-needed! :db)+ (let [fingerprint "latest-score-source-test/shared"]+ (try+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint)+ (data-complexity-score/record-score! fingerprint "appdb" {:meta {:label "appdb-row"}})+ (data-complexity-score/record-score! fingerprint "representation:abcd" {:meta {:label "representation-row"}})+ (is (= "appdb-row"+ (get-in (data-complexity-score/latest-score fingerprint) [:meta :label]))+ "default source=\"appdb\" skips past the newer representation-tagged row")+ (is (= "representation-row"+ (get-in (data-complexity-score/latest-score fingerprint "representation:abcd") [:meta :label]))+ "an explicit representation source still resolves its own row")+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))+(deftest ^:sequential run-scoring-persists-latest-score-snapshot-test(testing "every successful computation persists a fresh snapshot for the overview endpoint"(mt/initialize-if-needed! :db)@@ -996,9 +1018,11 @@(mt/with-temporary-setting-values [data-complexity-scoring-enabled true](mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] result)](#'task.complexity-score/run-scoring! "persist-test-fp")- (let [{:keys [id fingerprint score_data]} (data-complexity-score/latest-entry "persist-test-fp")]+ (let [{:keys [id fingerprint score_data source]} (data-complexity-score/latest-entry "persist-test-fp")](is (= result score_data))(is (= "persist-test-fp" fingerprint))+ (is (= "appdb" source)+ "cron-path rows must be stamped with source=\"appdb\"")(when before-id(is (> id before-id)"a new append-only snapshot should be written for each run")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/task/complexity_score_trimmer_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/task/complexity_score_trimmer_test.cljindex 6a1b9c127be..f1a4153b490 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/task/complexity_score_trimmer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/task/complexity_score_trimmer_test.clj@@ -27,6 +27,7 @@[label created-at](t2/insert! :model/DataComplexityScore{:fingerprint (str fingerprint-prefix label)+ :source "appdb":score_data {:label label}:created_at created-at}))diff --git a/resources/migrations/061/20260513_data_complexity_score_source.yaml b/resources/migrations/061/20260513_data_complexity_score_source.yamlnew file mode 100644index 00000000000..c169fdb4eaa--- /dev/null+++ b/resources/migrations/061/20260513_data_complexity_score_source.yaml@@ -0,0 +1,32 @@+## Add `source` column to data_complexity_score so we can distinguish rows produced from the live+## appdb (cron, API, CLI --source appdb) from rows produced by the CLI scoring a representation+## directory (CLI --source representation --write-to-appdb true).+##+## TODO (v62+): drop the `defaultValue: appdb` below. It exists only so old nodes that don't yet+## know about the `source` column can still insert during a rolling deploy. Once every deployment+## has rolled past v61, the default is dead weight and any insert that omits `source` is a bug we+## want to surface rather than silently label "appdb".++databaseChangeLog:+ - logicalFilePath: migrations/061_update_migrations.yaml+ - changeSet:+ id: v61.2026-05-13T10:00:00+ author: christruter+ comment: |+ Add NOT NULL source column with a default of 'appdb'. The default backfills existing rows+ and keeps old nodes' inserts safe during rolling deploys (they don't know to set source).+ changes:+ - addColumn:+ tableName: data_complexity_score+ columns:+ - column:+ name: source+ type: varchar(255)+ defaultValue: appdb+ constraints:+ nullable: false+ remarks: Provenance — "appdb" for cron/API/CLI-from-appdb, or "representation:<digest>" for CLI runs that scored a representation directory.+ rollback:+ - dropColumn:+ tableName: data_complexity_score+ columnName: sourcediff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex db427f3ff92..736a19c15e6 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -98,6 +98,31 @@[](reset! (:status mdb.connection/*application-db*) ::setup-finished))+(defn verify-application-db-connection!+ "Open a connection to the bound application DB and check its server version. Throws on failure.+ Does *not* run migrations or any encryption checks — use [[setup-db!]] for the full bootstrap.+ This is the public, no-argument hook for tools that need to confirm appdb connectivity without+ modifying the schema."+ []+ (mdb.setup/verify-db-connection (mdb.connection/db-type) (mdb.connection/data-source)))++(defn setup-db-without-migrations!+ "Like [[setup-db!]] but never runs migrations or mutates the schema. Idempotent: no-ops if the+ DB is already set up. Verifies connectivity, then marks the DB ready.++ For read-only tools (e.g. Data Complexity CLI) that need to query Toucan models against a live+ appdb without modifying it. Any schema mismatch surfaces as a runtime error at query time+ rather than during setup.++ Skips:+ - Liquibase migrations.+ - [[metabase.app-db.setup/check-encryption]], whose auto-encrypt branch can silently rewrite+ every encrypted `setting` row when an encryption key is configured."+ []+ (when-not (db-is-set-up?)+ (verify-application-db-connection!)+ (finish-db-setup!)))+(defn app-db"The Application database. A record, but use accessors [[db-type]], [[data-source]], etc to access. Alsoimplements [[javax.sql.DataSource]] directly, so you can call [[.getConnection]] on it directly."diff --git a/src/metabase/app_db/setup.clj b/src/metabase/app_db/setup.cljindex c529aa8aa17..0df21374ce2 100644--- a/src/metabase/app_db/setup.clj+++ b/src/metabase/app_db/setup.clj@@ -159,9 +159,10 @@(not (pos? (compare ((juxt :major :minor :patch) required-version)[major minor patch]))))))-(mu/defn- verify-db-connection+(mu/defn verify-db-connection"Test connection to application database with `data-source` and throw an exception if we have any troubles- connecting."+ connecting. Public so [[metabase.app-db.core/verify-application-db-connection!]] can call it from outside this+ namespace; most callers should use that wrapper instead of invoking this directly."[db-type :- :keyworddata-source :- (ms/InstanceOfClass javax.sql.DataSource)](log/info (u/format-color 'cyan "Verifying %s Database Connection ..." (name db-type)))diff --git a/src/metabase/core/bootstrap.clj b/src/metabase/core/bootstrap.cljindex eb79d3ed4cd..5c9b719b19c 100644--- a/src/metabase/core/bootstrap.clj+++ b/src/metabase/core/bootstrap.clj@@ -33,7 +33,9 @@;;; These modes don't need the full Metabase infrastructure (app-db, events,;;; etc.) and can run with minimal namespace loading for fast startup.;;;-;;; Usage: java -jar metabase.jar --mode checker [checker-specific args...]+;;; Usage:+;;; java -jar metabase.jar --mode checker [checker-specific args...]+;;; java -jar metabase.jar --mode complexity-score [scorer-specific args...];;; ===========================================================================#_{:clj-kondo/ignore [:discouraged-var]}@@ -48,13 +50,14 @@(let [startup (case mode;; schema checker was moved out to js. Perhaps this will get a long running server mode checker? Or;; perhaps just this one.- "checker" 'metabase-enterprise.checker.cli/entrypoint+ "checker" 'metabase-enterprise.checker.cli/entrypoint+ "complexity-score" 'metabase-enterprise.data-complexity-score.cli/entrypointnil)](if startup((requiring-resolve startup) args)(do (binding [*out* *err*](output! (str "Unknown mode: " mode))- (output! "Available modes: checker"))+ (output! "Available modes: checker, complexity-score"))(System/exit 1)))))(defn -main