The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Run the Data Complexity Score job weekly instead of daily (#75059)
Chris Truterdd086728158e
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.cljindex 9a2dae244fb..6b28bc2f3cc 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj@@ -75,7 +75,7 @@;; instead. In a clustered deployment the guard is per-node, so up to one pass per node can still;; run concurrently; we accept that since superuser API traffic is low-volume. The Quartz job;; already has its own concurrency control (`DisallowConcurrentExecution` + cluster lock for boot-;; emission), so we deliberately don't share this guard with the task path; a daily cron run that+;; emission), so we deliberately don't share this guard with the task path; a weekly cron run that;; coincided with an API call shouldn't be cancelled.(defonce ^:private ^java.util.concurrent.atomic.AtomicBoolean api-scoring-running?(java.util.concurrent.atomic.AtomicBoolean. false))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.cljindex 3a570bde5d6..40dca4afd1d 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj@@ -477,6 +477,11 @@Returns true when they are all successfully delivered.Returns false when tracking is disabled or any emission failed."[{:keys [library universe metabot meta]}]+ ;; TODO (Chris 2026-06-02) -- add an optional `computed_at` field to the data_complexity Snowplow schema+ ;; (1-0-1) and emit it, sourced from (:calculated-at meta) on the re-publish path and from compute-time+ ;; `now` on the fresh path.+ ;; Until then a re-published cached score (see [[republish-score!]]) is dated by Snowplow's collector+ ;; ingest time, overstating freshness by up to the cron cooldown window (12h).(let [base {:event :data_complexity_scoring:batch_id (str (random-uuid)):formula_version (:formula-version meta)@@ -490,6 +495,18 @@trueevents)))+(defn republish-score!+ "Re-emit an already-computed `score` map to Snowplow without recomputing — for retrying a publish+ that failed after the snapshot was already persisted. Returns the score carrying+ `::snowplow-published?` metadata, mirroring [[complexity-scores]] so callers share the+ `maybe-advance-last-fingerprint!` gate."+ [score]+ (with-meta score+ {::snowplow-published? (boolean (try+ (emit-snowplow! score)+ (catch Throwable t+ (log/warn t "Failed to re-publish cached complexity score to Snowplow"))))}))+(defn score-from-entities"Pure: compute the full complexity score from pre-built entity vectors and an embedder. No DBaccess, no Snowplow emission — suitable for callers that have already loaded their entitiesdiff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.cljindex 0c1a1fd7541..04f6f1746bb 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.data-complexity-score.models.data-complexity-score"Persistence for cached Data Complexity Score snapshots."(:require+ [metabase.app-db.core :as mdb][metabase.models.interface :as mi]+ [metabase.util.honey-sql-2 :as h2x][methodical.core :as methodical][toucan2.core :as t2]))@@ -32,6 +34,18 @@(some-> (latest-entry fingerprint source)score-with-calculated-at)))+(defn scored-within-cooldown?+ "True when a `source` snapshot for `fingerprint` was recorded within the last `cooldown-hours` hours.+ The cutoff is computed in database time, since `created_at` defaults to the DB `current_timestamp` —+ we never compare it against the app server's clock.+ Lets the scoring task skip a run that would only re-publish a still-fresh score."+ [fingerprint source cooldown-hours]+ (t2/exists? :model/DataComplexityScore+ {:where [:and+ [:= :fingerprint fingerprint]+ [:= :source source]+ [:>= :created_at (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- cooldown-hours) :hour)]]}))+(defn record-score!"Persist one append-only Data Complexity Score snapshot."[fingerprint source score]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.cljindex f52f76b5a3a..17c2338b977 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj@@ -1,5 +1,5 @@(ns metabase-enterprise.data-complexity-score.task.complexity-score- "Daily Quartz job that computes and publishes the Data Complexity Score."+ "Weekly Quartz job that computes and publishes the Data Complexity Score."(:require[clojure.edn :as edn][clojurewerkz.quartzite.jobs :as jobs]@@ -77,6 +77,16 @@;; claimant unblocks the next tick's retry without operator intervention.(def ^:private scoring-claim-ttl-ms (* 30 60 1000))+;; The weekly cron re-scores to capture catalog drift the fingerprint can't see, but a recompute for the+;; *same fingerprint* within this window — boot-time emission on a config change, an admin "recompute"+;; click, or a CLI appdb run — already produced an equivalent score, so the cron suppresses the recompute.+;; If that recent score was actually published the cron skips entirely; if the snapshot exists but its+;; publish lagged (`last-fingerprint` didn't advance) the cron re-emits the cached snapshot instead of+;; recomputing — see [[claim-scoring-run!]]'s `:mode`.+;; Fingerprint-scoped: a config change leaves no recent row for the new fingerprint, so the cron still+;; scores immediately.+(def ^:private cron-cooldown-hours 12)+(defn- now-ms [] (System/currentTimeMillis))(defn- parse-scoring-claim@@ -109,6 +119,12 @@`:require-fingerprint-change?` is true, when the live fingerprint still matches the lastsuccessfully-published one (boot's extra gate so restarts at an unchanged config don't re-emit).+ The returned claim carries a `:mode`. With `:cooldown-hours` set (the cron path) an appdb snapshot+ for the current fingerprint within that window means we skip the recompute, but the mode still+ distinguishes two cases: `:skip` (already published — `last-fingerprint` matches, no claim+ returned) versus `:republish` (scored but the publish lagged — re-emit the cached snapshot instead+ of recomputing). Outside the cooldown the mode is `:recompute`.+The returned claim must be passed back to [[release-scoring-claim!]] so the release is acompare-and-clear — a sibling that took over after a TTL-based expiry keeps its own claim.@@ -125,18 +141,29 @@Reads inside the lock bypass the in-process settings cache via `config/*disable-setting-cache*`so values freshly committed by sibling nodes are always visible — the cache is only refreshedperiodically and without bypass two nodes could both pass the check and both score."- [{:keys [require-fingerprint-change?]}]+ [{:keys [require-fingerprint-change? cooldown-hours]}](cluster-lock/with-cluster-lock {:lock ::complexity-score-run:timeout-seconds 10}(binding [config/*disable-setting-cache* true]- (let [current (current-fingerprint)]+ (let [current (current-fingerprint)+ last-fp (settings/data-complexity-scoring-last-fingerprint)+ recent? (boolean (and cooldown-hours+ (data-complexity-score/scored-within-cooldown? current "appdb" cooldown-hours)))+ ;; A scored-but-unpublished fingerprint (last-fp still lags) re-emits the cached snapshot+ ;; rather than recomputing — the cooldown suppresses the recompute, not the publish retry.+ mode (cond+ (and recent? (= current last-fp)) :skip+ recent? :republish+ :else :recompute)](when (and (settings/scoring-active?)+ (not= mode :skip)(or (not require-fingerprint-change?)- (not= current (settings/data-complexity-scoring-last-fingerprint)))+ (not= current last-fp))(not (scoring-claim-active?(parse-scoring-claim (settings/data-complexity-scoring-claim))current)))(let [claim {:fingerprint current+ :mode mode:claimed-at (now-ms):owner (str (random-uuid))}](settings/data-complexity-scoring-claim! (pr-str claim))@@ -160,20 +187,45 @@(log/warn t "Data Complexity Score: failed to clear scoring claim; it will expire via TTL"))))(defn- with-scoring-claim!- "Acquire a scoring claim, invoke `f` with the claim's fingerprint, then release the claim via+ "Acquire a scoring claim, invoke `f` with the claim map, then release the claim viacompare-and-clear. Returns nil when the claim couldn't be acquired (another path is scoring, orthe fingerprint-change gate didn't fire). `opts` pass through to [[claim-scoring-run!]]."[opts f](when-let [claim (claim-scoring-run! opts)](try- (f (:fingerprint claim))+ (f claim)(finally(release-scoring-claim! claim)))))+(defn- republish-cached-score!+ "Re-emit the latest cached appdb snapshot for `fingerprint` to Snowplow without recomputing, then+ advance `last-fingerprint` on a confirmed publish. Falls back to a full recompute if the snapshot+ the cooldown saw is already gone (manual delete / race)."+ [fingerprint]+ (if-let [cached (data-complexity-score/latest-score fingerprint "appdb")]+ (let [result (complexity/republish-score! cached)]+ (maybe-advance-last-fingerprint! fingerprint result)+ result)+ (run-scoring! fingerprint)))++(defn- run-claim!+ "Dispatch a claimed run by `:mode`:+ - `:republish` re-emits the cached snapshot+ - `:skip` is a no-op+ - anything else recomputes++ A `:skip` claim never reaches here — [[claim-scoring-run!]] returns nil in that mode — but the+ explicit branch keeps the dispatch total so a future caller can't silently recompute on it."+ [{:keys [mode fingerprint]}]+ (case mode+ :republish (republish-cached-score! fingerprint)+ :skip nil+ (run-scoring! fingerprint)))+(task/defjob ^{DisallowConcurrentExecution true:doc "Compute and publish the Data Complexity Score."}DataComplexityScoring [_ctx]- (with-scoring-claim! {} run-scoring!))+ (with-scoring-claim! {:cooldown-hours cron-cooldown-hours} run-claim!))(defn maybe-emit-boot-score!"Run one scoring pass at boot when the persisted fingerprint lags the live config. Shares the@@ -190,9 +242,9 @@[](try(with-scoring-claim! {:require-fingerprint-change? true}- (fn [fp]+ (fn [{:keys [fingerprint]}](log/info "Data Complexity Score: fingerprint changed, emitting boot-time score")- (run-scoring! fp)))+ (run-scoring! fingerprint)))(catch Throwable t(log/warn t "Data Complexity Score: boot-time emission failed"))))@@ -201,14 +253,14 @@(jobs/of-type DataComplexityScoring)(jobs/store-durably)(jobs/with-identity job-key)- (jobs/with-description "Data Complexity Score — daily telemetry"))- ;; 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.+ (jobs/with-description "Data Complexity Score — weekly telemetry"))+ ;; Sundays at 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.trigger (triggers/build(triggers/with-identity trigger-key)(triggers/for-job job-key)(triggers/with-schedule(cron/schedule- (cron/cron-schedule "0 17 3 * * ? *")+ (cron/cron-schedule "0 17 3 ? * SUN *")(cron/in-time-zone (java.util.TimeZone/getTimeZone "UTC"))(cron/with-misfire-handling-instruction-fire-and-proceed))))](task/schedule-task! job trigger)))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex 359f8e56b72..467e6527b60 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -1107,6 +1107,36 @@(finally(t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))+(deftest ^:sequential scored-within-cooldown-uses-db-time-test+ (testing "scored-within-cooldown? counts only same-fingerprint, same-source rows inside the window"+ ;; created_at defaults to the DB's current_timestamp, so the cutoff is computed in DB time. We+ ;; seed rows relative to real wall-clock (H2's clock ≈ the JVM's) with 1h/13h offsets — far from+ ;; the 12h boundary so there's no flake window.+ (mt/initialize-if-needed! :db)+ (let [fp "scored-within-test/fp"+ other "scored-within-test/other"+ recent (.minusHours (java.time.LocalDateTime/now) 1)+ stale (.minusHours (java.time.LocalDateTime/now) 13)+ insert! (fn [f source created-at]+ (t2/insert! :model/DataComplexityScore+ {:fingerprint f :source source :score_data {} :created_at created-at}))]+ (try+ (t2/delete! :model/DataComplexityScore :fingerprint [:in [fp other]])+ (insert! other "appdb" recent)+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))+ "a fresh row for a different fingerprint doesn't count")+ (insert! fp "appdb" stale)+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))+ "a same-fingerprint row older than the window doesn't count")+ (insert! fp "representation:abcd" recent)+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))+ "a fresh row from another source doesn't count")+ (insert! fp "appdb" recent)+ (is (data-complexity-score/scored-within-cooldown? fp "appdb" 12)+ "a fresh same-fingerprint appdb row counts")+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint [:in [fp other]]))))))+(deftest ^:sequential run-scoring-persists-latest-score-snapshot-test(testing "every successful computation persists a fresh snapshot for the overview endpoint"(mt/initialize-if-needed! :db)@@ -1125,9 +1155,8 @@(is (> id before-id)"a new append-only snapshot should be written for each run")))))))))-(deftest ^:sequential run-scoring-persists-fingerprint-only-on-successful-publish-test- (testing "fingerprint advances only when Snowplow accepted the event — failed publish must leave- the stale fingerprint in place so the next boot / cron retries"+(deftest ^:sequential run-scoring-advances-fingerprint-only-on-confirmed-publish-test+ (testing "fingerprint advances only when Snowplow accepted the event; any other outcome leaves it stale to retry"(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})](testing "successful publish → fingerprint advances to the claim's fingerprint"(mt/with-temporary-setting-values [data-complexity-scoring-enabled true@@ -1142,7 +1171,16 @@(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))](#'task.complexity-score/run-scoring! "fresh-fp")(is (= "stale" (settings/data-complexity-scoring-last-fingerprint))- "fingerprint preserved — next boot / cron will retry the emission")))))))+ "fingerprint preserved — next boot / cron will retry the emission"))))+ (testing "persistence throws → fingerprint stays stale so the failed cache write is retried"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))+ data-complexity-score/record-score! (fn [& _]+ (throw (RuntimeException. "db boom")))]+ (#'task.complexity-score/run-scoring! "fresh-fp")+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint preserved so the next boot / cron retries the failed cache write")))))))(deftest ^:sequential scoring-gate-matrix-test(testing "scoring runs iff :data-complexity-score premium feature OR deprecated setting is on"@@ -1176,18 +1214,6 @@(format "with-scoring-claim! should %sacquire a claim for [%s]"(if should-run? "" "NOT ") label)))))))))))-(deftest ^:sequential run-scoring-keeps-fingerprint-stale-when-persistence-fails-test- (testing "persistence is part of a successful run now — if the cache write fails we must retry"- (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true- data-complexity-scoring-last-fingerprint "stale"]- (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))- data-complexity-score/record-score! (fn [& _]- (throw (RuntimeException. "db boom")))]- (#'task.complexity-score/run-scoring! "fresh-fp")- (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))- "fingerprint preserved so the next boot / cron retries the failed cache write"))))))-(deftest ^:sequential maybe-emit-boot-score-only-advances-fingerprint-on-successful-publish-test(testing "boot-time emission never advances the last-successful fingerprint on failure — thesuccess fingerprint is separate from the in-progress scoring claim, so a scoring/publish@@ -1214,115 +1240,245 @@(is (= "" (settings/data-complexity-scoring-claim))"scoring claim released after successful run")))))))-(deftest ^:sequential maybe-emit-boot-score-skips-when-another-path-holds-active-claim-test- (testing "a fresh scoring claim (from a sibling node OR a concurrent cron tick) blocks duplicate- emission on this node, even when the last-successful fingerprint is stale — prevents- the boot and cron paths from both computing and publishing for the same fingerprint"+(deftest ^:sequential maybe-emit-boot-score-claim-protocol-test+ (testing "the shared scoring claim serializes boot-time emission against sibling nodes and cron ticks"(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]- (let [current-fp (#'task.complexity-score/current-fingerprint)- ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so- ;; it's well inside the TTL.- active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})- scoring-ran? (atom false)]- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true- data-complexity-scoring-last-fingerprint "stale"- data-complexity-scoring-claim active-claim]- (mt/with-dynamic-fn-redefs [complexity/complexity-scores- (fn [& _] (reset! scoring-ran? true) (stub-result true))]- (task.complexity-score/maybe-emit-boot-score!)- (is (false? @scoring-ran?)- "scoring skipped because another path already claimed the current fingerprint")- (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))- "fingerprint untouched when the claim is skipped")- (is (= active-claim (settings/data-complexity-scoring-claim))- "other path's claim is preserved (we never took it, so we don't clear it)")))))))+ (testing "a fresh claim (sibling node or concurrent cron tick) blocks duplicate emission even when last-fp is stale"+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so+ ;; it's well inside the TTL.+ active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim active-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (false? @scoring-ran?)+ "scoring skipped because another path already claimed the current fingerprint")+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint untouched when the claim is skipped")+ (is (= active-claim (settings/data-complexity-scoring-claim))+ "other path's claim is preserved (we never took it, so we don't clear it)")))))+ (testing "an expired (TTL-exceeded) claim is treated as orphaned — scoring re-claims and proceeds"+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.+ expired-claim (pr-str {:fingerprint current-fp+ :claimed-at (- (#'task.complexity-score/now-ms)+ (* 60 60 1000))})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim expired-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (true? @scoring-ran?)+ "scoring ran because the prior claim had aged past the TTL")+ (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint advanced on successful publish after re-claim")))))+ (testing "a sibling re-claim during our run is not wiped by our release (compare-and-clear on :owner)"+ ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a+ ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min+ ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.+ (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)+ :claimed-at (System/currentTimeMillis)+ :owner "sibling-node-owner-token"})]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _]+ (settings/data-complexity-scoring-claim! sibling-claim)+ (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (= sibling-claim (settings/data-complexity-scoring-claim))+ "replacement claim preserved — our release was a compare-and-clear and the owners didn't match"))))))))-(deftest ^:sequential maybe-emit-boot-score-reclaims-when-prior-claim-has-expired-test- (testing "an expired (TTL-exceeded) scoring claim must not permanently suppress scoring — a- crashed or orphaned claim from a prior run should be replaced and scoring proceed"+(deftest ^:sequential claim-scoring-run-mode-selection-test+ (testing "the cron cooldown gate resolves to :skip / :republish / :recompute by publish state"(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]- (let [current-fp (#'task.complexity-score/current-fingerprint)- ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.- expired-claim (pr-str {:fingerprint current-fp- :claimed-at (- (#'task.complexity-score/now-ms)- (* 60 60 1000))})- scoring-ran? (atom false)]- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true- data-complexity-scoring-last-fingerprint "stale"- data-complexity-scoring-claim expired-claim]- (mt/with-dynamic-fn-redefs [complexity/complexity-scores- (fn [& _] (reset! scoring-ran? true) (stub-result true))]- (task.complexity-score/maybe-emit-boot-score!)- (is (true? @scoring-ran?)- "scoring ran because the prior claim had aged past the TTL")- (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))- "fingerprint advanced on successful publish after re-claim")))))))+ (let [current (#'task.complexity-score/current-fingerprint)]+ (testing "within cooldown AND already published (last-fp = current) → :skip, no claim acquired"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint current+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)]+ (is (nil? (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))+ (testing "within cooldown but publish lagged (last-fp ≠ current) → :republish"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)]+ (is (=? {:mode :republish :fingerprint current}+ (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))+ (testing "outside the cooldown → :recompute"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly false)]+ (is (=? {:mode :recompute :fingerprint current}+ (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))))))-(deftest ^:sequential maybe-emit-boot-score-does-not-clear-sibling-claim-after-ttl-takeover-test- (testing "if our run outlives the TTL and another path legitimately re-claims, the original- claimant's release-scoring-claim! must NOT wipe the replacement claim — doing so would- reopen the duplicate-publish race (a third node/path would see no claim and run again)"+(deftest ^:sequential republish-cached-score-test+ (testing "the re-publish path re-emits the cached snapshot without recomputing"+ (mt/initialize-if-needed! :db)(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]- ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a- ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min- ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.- (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)- :claimed-at (System/currentTimeMillis)- :owner "sibling-node-owner-token"})]+ (testing "cached snapshot + confirmed publish → fingerprint advances, no recompute"(mt/with-temporary-setting-values [data-complexity-scoring-enabled true- data-complexity-scoring-last-fingerprint "stale"- data-complexity-scoring-claim ""]- (mt/with-dynamic-fn-redefs [complexity/complexity-scores- (fn [& _]- (settings/data-complexity-scoring-claim! sibling-claim)- (stub-result true))]- (task.complexity-score/maybe-emit-boot-score!)- (is (= sibling-claim (settings/data-complexity-scoring-claim))- "replacement claim preserved — our release was a compare-and-clear and the owners didn't match")))))))+ data-complexity-scoring-last-fingerprint "stale"]+ (try+ (data-complexity-score/record-score! "republish-fp" "appdb" (stub-result false))+ (let [recompute? (atom false)+ handed (atom nil)]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))+ complexity/republish-score! (fn [s]+ (reset! handed s)+ (with-meta s {::complexity/snowplow-published? true}))]+ (#'task.complexity-score/republish-cached-score! "republish-fp")+ (is (false? @recompute?) "must re-publish the cached score, never recompute")+ (is (some? @handed) "cached snapshot handed to republish-score!")+ (is (= "republish-fp" (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint advances on a confirmed re-publish")))+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint "republish-fp")))))+ (testing "failed re-publish leaves the fingerprint stale for the next retry"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"]+ (try+ (data-complexity-score/record-score! "republish-fail-fp" "appdb" (stub-result false))+ (mt/with-dynamic-fn-redefs [complexity/republish-score! (fn [s] (with-meta s {::complexity/snowplow-published? false}))]+ (#'task.complexity-score/republish-cached-score! "republish-fail-fp")+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint preserved when the re-publish fails"))+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint "republish-fail-fp")))))+ (testing "cooldown saw a row that's since gone → falls back to a full recompute"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"]+ (let [recompute? (atom false)]+ (mt/with-dynamic-fn-redefs [data-complexity-score/latest-score (constantly nil)+ complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))]+ (#'task.complexity-score/republish-cached-score! "missing-fp")+ (is (true? @recompute?) "a missing snapshot must trigger a recompute"))))))))-(deftest ^:sequential cron-skips-when-boot-run-holds-active-claim-test- (testing "the cron job's scoring call skips when the boot-time run is already mid-flight for the- current fingerprint — the shared claim protocol serializes the two paths so they can't- both compute and publish in parallel"+(deftest ^:sequential cron-run-claim-path-test+ (testing "the cron path — with-scoring-claim! + run-claim!, exactly what the Quartz job body calls"+ (mt/initialize-if-needed! :db)(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]- (let [current-fp (#'task.complexity-score/current-fingerprint)- boot-claim (pr-str {:fingerprint current-fp- :claimed-at (#'task.complexity-score/now-ms)- :owner "boot-path-owner-token"})- scoring-ran? (atom false)]- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true- data-complexity-scoring-last-fingerprint "stale"- data-complexity-scoring-claim boot-claim]- (mt/with-dynamic-fn-redefs [complexity/complexity-scores- (fn [& _] (reset! scoring-ran? true) (stub-result true))]- ;; Exercise the cron's code path via the shared helper — this is exactly what the- ;; Quartz job body calls. Re-sampling current-fingerprint does not matter because the- ;; live value matches the one the boot path claimed.- (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-scoring!)- (is (false? @scoring-ran?)- "cron tick skipped because the boot run holds the scoring claim")- (is (= boot-claim (settings/data-complexity-scoring-claim))- "boot's claim preserved — cron never took it, so it doesn't clear it")))))))+ (testing "skips when a boot-time run already holds an active claim for the current fingerprint"+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ boot-claim (pr-str {:fingerprint current-fp+ :claimed-at (#'task.complexity-score/now-ms)+ :owner "boot-path-owner-token"})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim boot-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-claim!)+ (is (false? @scoring-ran?)+ "cron tick skipped because the boot run holds the scoring claim")+ (is (= boot-claim (settings/data-complexity-scoring-claim))+ "boot's claim preserved — cron never took it, so it doesn't clear it")))))+ (testing "regression: a scored-but-unpublished snapshot within the cooldown re-publishes instead of skipping for a week"+ (let [current (#'task.complexity-score/current-fingerprint)+ recompute? (atom false)+ republished? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ ;; publish lagged — last-fp never advanced to the scored fingerprint+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (try+ (data-complexity-score/record-score! current "appdb" (stub-result false))+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)+ complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))+ complexity/republish-score! (fn [s]+ (reset! republished? true)+ (with-meta s {::complexity/snowplow-published? true}))]+ (#'task.complexity-score/with-scoring-claim! {:cooldown-hours 12} #'task.complexity-score/run-claim!)+ (is (true? @republished?) "re-published the cached snapshot")+ (is (false? @recompute?) "did not recompute")+ (is (= current (settings/data-complexity-scoring-last-fingerprint))+ "fingerprint advanced after the re-publish — telemetry no longer stalled for a week"))+ ;; The row is written under the *real* current-fingerprint with created_at=now — delete it+ ;; so a later :sequential test exercising the real cooldown path doesn't see a spurious row.+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint current :source "appdb")))))))))-(deftest ^:sequential complexity-scores-tags-publish-success-on-result-test- (testing "complexity-scores stamps publish success/failure via metadata for schedule/boot callers"- (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs- (constantly {:library [] :universe [] :metabot []})]- (testing "successful publish → ::snowplow-published? true"+(deftest ^:sequential publish-tagging-test+ (testing "::snowplow-published? metadata records publish success/failure for schedule/boot callers"+ (testing "complexity-scores (fresh compute + publish)"+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [] :universe [] :metabot []})]+ (testing "successful publish → ::snowplow-published? true"+ (snowplow-test/with-fake-snowplow-collector+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result)))))))+ (testing "publish throw → ::snowplow-published? false (and the result is still returned)"+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result)))))))+ (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"+ ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would+ ;; look indistinguishable from a successful publish and the fingerprint would advance,+ ;; silently suppressing the boot-time retry once tracking is turned back on.+ (mt/with-temporary-setting-values [anon-tracking-enabled false]+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result)))))))))+ (testing "republish-score! (re-emit cached, no recompute) mirrors that tagging"+ (let [score {:library {:score 0 :components {}}+ :universe {:score 0 :components {}}+ :metabot {:score 0 :components {}}+ :meta {:formula-version 1 :synonym-threshold 0.8 :weights {}}}]+ (testing "publish succeeds → tagged true"+ (snowplow-test/with-fake-snowplow-collector+ (is (true? (::complexity/snowplow-published? (meta (complexity/republish-score! score)))))))+ (testing "publish throws → tagged false, score still returned unchanged"+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]+ (let [result (complexity/republish-score! score)]+ (is (false? (::complexity/snowplow-published? (meta result))))+ (is (= score result) "the score value is returned untouched"))))))))++(deftest ^:sequential republish-from-cached-snapshot-round-trip-test+ (testing "the real latest-score → republish-score! → emit-snowplow! round-trip emits events from a cached snapshot"+ (mt/initialize-if-needed! :db)+ (let [fingerprint "republish-round-trip-test/fp"+ ;; :meta carries keyword values (:text-variant and nested :embedding-model) that JSON storage+ ;; round-trips back as strings — the fragile bit emit-snowplow! must keep tolerating, and the+ ;; whole point of the republish-from-cache path this commit adds.+ score {:library {:score 1 :components {}}+ :universe {:score 2 :components {}}+ :metabot {:score 3 :components {}}+ :meta {:formula-version 1+ :synonym-threshold 0.8+ :weights {}+ :text-variant :names-split+ :embedding-model {:provider :ai-service :model-name "minilm" :model-dimensions 384}}}]+ (try+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint)+ (data-complexity-score/record-score! fingerprint "appdb" score)(snowplow-test/with-fake-snowplow-collector- (let [result (complexity/complexity-scores :embedder nil)]- (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?- (meta result)))))))- (testing "publish throw → ::snowplow-published? false (and the result is still returned)"- (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]- (let [result (complexity/complexity-scores :embedder nil)]- (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?- (meta result)))))))- (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"- ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would- ;; look indistinguishable from a successful publish and the fingerprint would advance,- ;; silently suppressing the boot-time retry once tracking is turned back on.- (mt/with-temporary-setting-values [anon-tracking-enabled false]- (let [result (complexity/complexity-scores :embedder nil)]- (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?- (meta result))))))))))+ (snowplow-test/pop-event-data-and-user-id!) ; drain startup/setting events+ (let [cached (data-complexity-score/latest-score fingerprint "appdb")+ result (complexity/republish-score! cached)+ events (complexity-events!)]+ (is (true? (::complexity/snowplow-published? (meta result)))+ "publish reported success")+ (is (= 3 (count events))+ "library, universe, and metabot totals re-emitted from the cached snapshot")+ (is (= "names_split" (get-in (first events) ["parameters" "text_variant"]))+ "keyword :meta values survive the JSON round-trip into the emitted parameters")+ ;; Round-tripped :embedding-model flows through the recursive snake-keys walk. Its+ ;; `:provider` value comes back as the string "ai-service", so it stays "ai-service" — a+ ;; fresh in-memory `:ai-service` keyword would instead snake to "ai_service".+ (is (= {"provider" "ai-service" "model_name" "minilm" "model_dimensions" 384}+ (get-in (first events) ["parameters" "embedding_model"]))+ "nested :embedding-model survives the recursive snake-keys walk over round-tripped data")))+ (finally+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))