Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Run the Data Complexity Score job weekly instead of daily (#75059)

Chris Truterdd086728158e
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
index 9a2dae244fb..6b28bc2f3cc 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
@@ -75,7 +75,7 @@
;; instead. In a clustered deployment the guard is per-node, so up to one pass per node can still
;; run concurrently; we accept that since superuser API traffic is low-volume. The Quartz job
;; already has its own concurrency control (`DisallowConcurrentExecution` + cluster lock for boot
-;; emission), so we deliberately don't share this guard with the task path; a daily cron run that
+;; emission), so we deliberately don't share this guard with the task path; a weekly cron run that
;; coincided with an API call shouldn't be cancelled.
(defonce ^:private ^java.util.concurrent.atomic.AtomicBoolean api-scoring-running?
(java.util.concurrent.atomic.AtomicBoolean. false))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
index 3a570bde5d6..40dca4afd1d 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -477,6 +477,11 @@
Returns true when they are all successfully delivered.
Returns false when tracking is disabled or any emission failed."
[{:keys [library universe metabot meta]}]
+ ;; TODO (Chris 2026-06-02) -- add an optional `computed_at` field to the data_complexity Snowplow schema
+ ;; (1-0-1) and emit it, sourced from (:calculated-at meta) on the re-publish path and from compute-time
+ ;; `now` on the fresh path.
+ ;; Until then a re-published cached score (see [[republish-score!]]) is dated by Snowplow's collector
+ ;; ingest time, overstating freshness by up to the cron cooldown window (12h).
(let [base {:event :data_complexity_scoring
:batch_id (str (random-uuid))
:formula_version (:formula-version meta)
@@ -490,6 +495,18 @@
true
events)))
+(defn republish-score!
+ "Re-emit an already-computed `score` map to Snowplow without recomputing — for retrying a publish
+ that failed after the snapshot was already persisted. Returns the score carrying
+ `::snowplow-published?` metadata, mirroring [[complexity-scores]] so callers share the
+ `maybe-advance-last-fingerprint!` gate."
+ [score]
+ (with-meta score
+ {::snowplow-published? (boolean (try
+ (emit-snowplow! score)
+ (catch Throwable t
+ (log/warn t "Failed to re-publish cached complexity score to Snowplow"))))}))
+
(defn score-from-entities
"Pure: compute the full complexity score from pre-built entity vectors and an embedder. No DB
access, no Snowplow emission — suitable for callers that have already loaded their entities
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj
index 0c1a1fd7541..04f6f1746bb 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/models/data_complexity_score.clj
@@ -1,7 +1,9 @@
(ns metabase-enterprise.data-complexity-score.models.data-complexity-score
"Persistence for cached Data Complexity Score snapshots."
(:require
+ [metabase.app-db.core :as mdb]
[metabase.models.interface :as mi]
+ [metabase.util.honey-sql-2 :as h2x]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -32,6 +34,18 @@
(some-> (latest-entry fingerprint source)
score-with-calculated-at)))
+(defn scored-within-cooldown?
+ "True when a `source` snapshot for `fingerprint` was recorded within the last `cooldown-hours` hours.
+ The cutoff is computed in database time, since `created_at` defaults to the DB `current_timestamp` —
+ we never compare it against the app server's clock.
+ Lets the scoring task skip a run that would only re-publish a still-fresh score."
+ [fingerprint source cooldown-hours]
+ (t2/exists? :model/DataComplexityScore
+ {:where [:and
+ [:= :fingerprint fingerprint]
+ [:= :source source]
+ [:>= :created_at (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- cooldown-hours) :hour)]]}))
+
(defn record-score!
"Persist one append-only Data Complexity Score snapshot."
[fingerprint source score]
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
index f52f76b5a3a..17c2338b977 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
@@ -1,5 +1,5 @@
(ns metabase-enterprise.data-complexity-score.task.complexity-score
- "Daily Quartz job that computes and publishes the Data Complexity Score."
+ "Weekly Quartz job that computes and publishes the Data Complexity Score."
(:require
[clojure.edn :as edn]
[clojurewerkz.quartzite.jobs :as jobs]
@@ -77,6 +77,16 @@
;; claimant unblocks the next tick's retry without operator intervention.
(def ^:private scoring-claim-ttl-ms (* 30 60 1000))
+;; The weekly cron re-scores to capture catalog drift the fingerprint can't see, but a recompute for the
+;; *same fingerprint* within this window — boot-time emission on a config change, an admin "recompute"
+;; click, or a CLI appdb run — already produced an equivalent score, so the cron suppresses the recompute.
+;; If that recent score was actually published the cron skips entirely; if the snapshot exists but its
+;; publish lagged (`last-fingerprint` didn't advance) the cron re-emits the cached snapshot instead of
+;; recomputing — see [[claim-scoring-run!]]'s `:mode`.
+;; Fingerprint-scoped: a config change leaves no recent row for the new fingerprint, so the cron still
+;; scores immediately.
+(def ^:private cron-cooldown-hours 12)
+
(defn- now-ms [] (System/currentTimeMillis))
(defn- parse-scoring-claim
@@ -109,6 +119,12 @@
`:require-fingerprint-change?` is true, when the live fingerprint still matches the last
successfully-published one (boot's extra gate so restarts at an unchanged config don't re-emit).
+ The returned claim carries a `:mode`. With `:cooldown-hours` set (the cron path) an appdb snapshot
+ for the current fingerprint within that window means we skip the recompute, but the mode still
+ distinguishes two cases: `:skip` (already published — `last-fingerprint` matches, no claim
+ returned) versus `:republish` (scored but the publish lagged — re-emit the cached snapshot instead
+ of recomputing). Outside the cooldown the mode is `:recompute`.
+
The returned claim must be passed back to [[release-scoring-claim!]] so the release is a
compare-and-clear — a sibling that took over after a TTL-based expiry keeps its own claim.
@@ -125,18 +141,29 @@
Reads inside the lock bypass the in-process settings cache via `config/*disable-setting-cache*`
so values freshly committed by sibling nodes are always visible — the cache is only refreshed
periodically and without bypass two nodes could both pass the check and both score."
- [{:keys [require-fingerprint-change?]}]
+ [{:keys [require-fingerprint-change? cooldown-hours]}]
(cluster-lock/with-cluster-lock {:lock ::complexity-score-run
:timeout-seconds 10}
(binding [config/*disable-setting-cache* true]
- (let [current (current-fingerprint)]
+ (let [current (current-fingerprint)
+ last-fp (settings/data-complexity-scoring-last-fingerprint)
+ recent? (boolean (and cooldown-hours
+ (data-complexity-score/scored-within-cooldown? current "appdb" cooldown-hours)))
+ ;; A scored-but-unpublished fingerprint (last-fp still lags) re-emits the cached snapshot
+ ;; rather than recomputing — the cooldown suppresses the recompute, not the publish retry.
+ mode (cond
+ (and recent? (= current last-fp)) :skip
+ recent? :republish
+ :else :recompute)]
(when (and (settings/scoring-active?)
+ (not= mode :skip)
(or (not require-fingerprint-change?)
- (not= current (settings/data-complexity-scoring-last-fingerprint)))
+ (not= current last-fp))
(not (scoring-claim-active?
(parse-scoring-claim (settings/data-complexity-scoring-claim))
current)))
(let [claim {:fingerprint current
+ :mode mode
:claimed-at (now-ms)
:owner (str (random-uuid))}]
(settings/data-complexity-scoring-claim! (pr-str claim))
@@ -160,20 +187,45 @@
(log/warn t "Data Complexity Score: failed to clear scoring claim; it will expire via TTL"))))
(defn- with-scoring-claim!
- "Acquire a scoring claim, invoke `f` with the claim's fingerprint, then release the claim via
+ "Acquire a scoring claim, invoke `f` with the claim map, then release the claim via
compare-and-clear. Returns nil when the claim couldn't be acquired (another path is scoring, or
the fingerprint-change gate didn't fire). `opts` pass through to [[claim-scoring-run!]]."
[opts f]
(when-let [claim (claim-scoring-run! opts)]
(try
- (f (:fingerprint claim))
+ (f claim)
(finally
(release-scoring-claim! claim)))))
+(defn- republish-cached-score!
+ "Re-emit the latest cached appdb snapshot for `fingerprint` to Snowplow without recomputing, then
+ advance `last-fingerprint` on a confirmed publish. Falls back to a full recompute if the snapshot
+ the cooldown saw is already gone (manual delete / race)."
+ [fingerprint]
+ (if-let [cached (data-complexity-score/latest-score fingerprint "appdb")]
+ (let [result (complexity/republish-score! cached)]
+ (maybe-advance-last-fingerprint! fingerprint result)
+ result)
+ (run-scoring! fingerprint)))
+
+(defn- run-claim!
+ "Dispatch a claimed run by `:mode`:
+ - `:republish` re-emits the cached snapshot
+ - `:skip` is a no-op
+ - anything else recomputes
+
+ A `:skip` claim never reaches here — [[claim-scoring-run!]] returns nil in that mode — but the
+ explicit branch keeps the dispatch total so a future caller can't silently recompute on it."
+ [{:keys [mode fingerprint]}]
+ (case mode
+ :republish (republish-cached-score! fingerprint)
+ :skip nil
+ (run-scoring! fingerprint)))
+
(task/defjob ^{DisallowConcurrentExecution true
:doc "Compute and publish the Data Complexity Score."}
DataComplexityScoring [_ctx]
- (with-scoring-claim! {} run-scoring!))
+ (with-scoring-claim! {:cooldown-hours cron-cooldown-hours} run-claim!))
(defn maybe-emit-boot-score!
"Run one scoring pass at boot when the persisted fingerprint lags the live config. Shares the
@@ -190,9 +242,9 @@
[]
(try
(with-scoring-claim! {:require-fingerprint-change? true}
- (fn [fp]
+ (fn [{:keys [fingerprint]}]
(log/info "Data Complexity Score: fingerprint changed, emitting boot-time score")
- (run-scoring! fp)))
+ (run-scoring! fingerprint)))
(catch Throwable t
(log/warn t "Data Complexity Score: boot-time emission failed"))))
@@ -201,14 +253,14 @@
(jobs/of-type DataComplexityScoring)
(jobs/store-durably)
(jobs/with-identity job-key)
- (jobs/with-description "Data Complexity Score — daily telemetry"))
- ;; 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.
+ (jobs/with-description "Data Complexity Score — weekly telemetry"))
+ ;; Sundays at 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.
trigger (triggers/build
(triggers/with-identity trigger-key)
(triggers/for-job job-key)
(triggers/with-schedule
(cron/schedule
- (cron/cron-schedule "0 17 3 * * ? *")
+ (cron/cron-schedule "0 17 3 ? * SUN *")
(cron/in-time-zone (java.util.TimeZone/getTimeZone "UTC"))
(cron/with-misfire-handling-instruction-fire-and-proceed))))]
(task/schedule-task! job trigger)))
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index 359f8e56b72..467e6527b60 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -1107,6 +1107,36 @@
(finally
(t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))
+(deftest ^:sequential scored-within-cooldown-uses-db-time-test
+ (testing "scored-within-cooldown? counts only same-fingerprint, same-source rows inside the window"
+ ;; created_at defaults to the DB's current_timestamp, so the cutoff is computed in DB time. We
+ ;; seed rows relative to real wall-clock (H2's clock ≈ the JVM's) with 1h/13h offsets — far from
+ ;; the 12h boundary so there's no flake window.
+ (mt/initialize-if-needed! :db)
+ (let [fp "scored-within-test/fp"
+ other "scored-within-test/other"
+ recent (.minusHours (java.time.LocalDateTime/now) 1)
+ stale (.minusHours (java.time.LocalDateTime/now) 13)
+ insert! (fn [f source created-at]
+ (t2/insert! :model/DataComplexityScore
+ {:fingerprint f :source source :score_data {} :created_at created-at}))]
+ (try
+ (t2/delete! :model/DataComplexityScore :fingerprint [:in [fp other]])
+ (insert! other "appdb" recent)
+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))
+ "a fresh row for a different fingerprint doesn't count")
+ (insert! fp "appdb" stale)
+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))
+ "a same-fingerprint row older than the window doesn't count")
+ (insert! fp "representation:abcd" recent)
+ (is (not (data-complexity-score/scored-within-cooldown? fp "appdb" 12))
+ "a fresh row from another source doesn't count")
+ (insert! fp "appdb" recent)
+ (is (data-complexity-score/scored-within-cooldown? fp "appdb" 12)
+ "a fresh same-fingerprint appdb row counts")
+ (finally
+ (t2/delete! :model/DataComplexityScore :fingerprint [:in [fp other]]))))))
+
(deftest ^:sequential run-scoring-persists-latest-score-snapshot-test
(testing "every successful computation persists a fresh snapshot for the overview endpoint"
(mt/initialize-if-needed! :db)
@@ -1125,9 +1155,8 @@
(is (> id before-id)
"a new append-only snapshot should be written for each run")))))))))
-(deftest ^:sequential run-scoring-persists-fingerprint-only-on-successful-publish-test
- (testing "fingerprint advances only when Snowplow accepted the event — failed publish must leave
- the stale fingerprint in place so the next boot / cron retries"
+(deftest ^:sequential run-scoring-advances-fingerprint-only-on-confirmed-publish-test
+ (testing "fingerprint advances only when Snowplow accepted the event; any other outcome leaves it stale to retry"
(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
(testing "successful publish → fingerprint advances to the claim's fingerprint"
(mt/with-temporary-setting-values [data-complexity-scoring-enabled true
@@ -1142,7 +1171,16 @@
(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]
(#'task.complexity-score/run-scoring! "fresh-fp")
(is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
- "fingerprint preserved — next boot / cron will retry the emission")))))))
+ "fingerprint preserved — next boot / cron will retry the emission"))))
+ (testing "persistence throws → fingerprint stays stale so the failed cache write is retried"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))
+ data-complexity-score/record-score! (fn [& _]
+ (throw (RuntimeException. "db boom")))]
+ (#'task.complexity-score/run-scoring! "fresh-fp")
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint preserved so the next boot / cron retries the failed cache write")))))))
(deftest ^:sequential scoring-gate-matrix-test
(testing "scoring runs iff :data-complexity-score premium feature OR deprecated setting is on"
@@ -1176,18 +1214,6 @@
(format "with-scoring-claim! should %sacquire a claim for [%s]"
(if should-run? "" "NOT ") label)))))))))))
-(deftest ^:sequential run-scoring-keeps-fingerprint-stale-when-persistence-fails-test
- (testing "persistence is part of a successful run now — if the cache write fails we must retry"
- (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
- data-complexity-scoring-last-fingerprint "stale"]
- (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))
- data-complexity-score/record-score! (fn [& _]
- (throw (RuntimeException. "db boom")))]
- (#'task.complexity-score/run-scoring! "fresh-fp")
- (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
- "fingerprint preserved so the next boot / cron retries the failed cache write"))))))
-
(deftest ^:sequential maybe-emit-boot-score-only-advances-fingerprint-on-successful-publish-test
(testing "boot-time emission never advances the last-successful fingerprint on failure — the
success fingerprint is separate from the in-progress scoring claim, so a scoring/publish
@@ -1214,115 +1240,245 @@
(is (= "" (settings/data-complexity-scoring-claim))
"scoring claim released after successful run")))))))
-(deftest ^:sequential maybe-emit-boot-score-skips-when-another-path-holds-active-claim-test
- (testing "a fresh scoring claim (from a sibling node OR a concurrent cron tick) blocks duplicate
- emission on this node, even when the last-successful fingerprint is stale — prevents
- the boot and cron paths from both computing and publishing for the same fingerprint"
+(deftest ^:sequential maybe-emit-boot-score-claim-protocol-test
+ (testing "the shared scoring claim serializes boot-time emission against sibling nodes and cron ticks"
(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
- (let [current-fp (#'task.complexity-score/current-fingerprint)
- ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so
- ;; it's well inside the TTL.
- active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})
- scoring-ran? (atom false)]
- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
- data-complexity-scoring-last-fingerprint "stale"
- data-complexity-scoring-claim active-claim]
- (mt/with-dynamic-fn-redefs [complexity/complexity-scores
- (fn [& _] (reset! scoring-ran? true) (stub-result true))]
- (task.complexity-score/maybe-emit-boot-score!)
- (is (false? @scoring-ran?)
- "scoring skipped because another path already claimed the current fingerprint")
- (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
- "fingerprint untouched when the claim is skipped")
- (is (= active-claim (settings/data-complexity-scoring-claim))
- "other path's claim is preserved (we never took it, so we don't clear it)")))))))
+ (testing "a fresh claim (sibling node or concurrent cron tick) blocks duplicate emission even when last-fp is stale"
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so
+ ;; it's well inside the TTL.
+ active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim active-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (false? @scoring-ran?)
+ "scoring skipped because another path already claimed the current fingerprint")
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint untouched when the claim is skipped")
+ (is (= active-claim (settings/data-complexity-scoring-claim))
+ "other path's claim is preserved (we never took it, so we don't clear it)")))))
+ (testing "an expired (TTL-exceeded) claim is treated as orphaned — scoring re-claims and proceeds"
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.
+ expired-claim (pr-str {:fingerprint current-fp
+ :claimed-at (- (#'task.complexity-score/now-ms)
+ (* 60 60 1000))})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim expired-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (true? @scoring-ran?)
+ "scoring ran because the prior claim had aged past the TTL")
+ (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint advanced on successful publish after re-claim")))))
+ (testing "a sibling re-claim during our run is not wiped by our release (compare-and-clear on :owner)"
+ ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a
+ ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min
+ ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.
+ (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)
+ :claimed-at (System/currentTimeMillis)
+ :owner "sibling-node-owner-token"})]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _]
+ (settings/data-complexity-scoring-claim! sibling-claim)
+ (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (= sibling-claim (settings/data-complexity-scoring-claim))
+ "replacement claim preserved — our release was a compare-and-clear and the owners didn't match"))))))))
-(deftest ^:sequential maybe-emit-boot-score-reclaims-when-prior-claim-has-expired-test
- (testing "an expired (TTL-exceeded) scoring claim must not permanently suppress scoring — a
- crashed or orphaned claim from a prior run should be replaced and scoring proceed"
+(deftest ^:sequential claim-scoring-run-mode-selection-test
+ (testing "the cron cooldown gate resolves to :skip / :republish / :recompute by publish state"
(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
- (let [current-fp (#'task.complexity-score/current-fingerprint)
- ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.
- expired-claim (pr-str {:fingerprint current-fp
- :claimed-at (- (#'task.complexity-score/now-ms)
- (* 60 60 1000))})
- scoring-ran? (atom false)]
- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
- data-complexity-scoring-last-fingerprint "stale"
- data-complexity-scoring-claim expired-claim]
- (mt/with-dynamic-fn-redefs [complexity/complexity-scores
- (fn [& _] (reset! scoring-ran? true) (stub-result true))]
- (task.complexity-score/maybe-emit-boot-score!)
- (is (true? @scoring-ran?)
- "scoring ran because the prior claim had aged past the TTL")
- (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))
- "fingerprint advanced on successful publish after re-claim")))))))
+ (let [current (#'task.complexity-score/current-fingerprint)]
+ (testing "within cooldown AND already published (last-fp = current) → :skip, no claim acquired"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint current
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)]
+ (is (nil? (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))
+ (testing "within cooldown but publish lagged (last-fp ≠ current) → :republish"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)]
+ (is (=? {:mode :republish :fingerprint current}
+ (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))
+ (testing "outside the cooldown → :recompute"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly false)]
+ (is (=? {:mode :recompute :fingerprint current}
+ (#'task.complexity-score/claim-scoring-run! {:cooldown-hours 12}))))))))))
-(deftest ^:sequential maybe-emit-boot-score-does-not-clear-sibling-claim-after-ttl-takeover-test
- (testing "if our run outlives the TTL and another path legitimately re-claims, the original
- claimant's release-scoring-claim! must NOT wipe the replacement claim — doing so would
- reopen the duplicate-publish race (a third node/path would see no claim and run again)"
+(deftest ^:sequential republish-cached-score-test
+ (testing "the re-publish path re-emits the cached snapshot without recomputing"
+ (mt/initialize-if-needed! :db)
(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
- ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a
- ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min
- ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.
- (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)
- :claimed-at (System/currentTimeMillis)
- :owner "sibling-node-owner-token"})]
+ (testing "cached snapshot + confirmed publish → fingerprint advances, no recompute"
(mt/with-temporary-setting-values [data-complexity-scoring-enabled true
- data-complexity-scoring-last-fingerprint "stale"
- data-complexity-scoring-claim ""]
- (mt/with-dynamic-fn-redefs [complexity/complexity-scores
- (fn [& _]
- (settings/data-complexity-scoring-claim! sibling-claim)
- (stub-result true))]
- (task.complexity-score/maybe-emit-boot-score!)
- (is (= sibling-claim (settings/data-complexity-scoring-claim))
- "replacement claim preserved — our release was a compare-and-clear and the owners didn't match")))))))
+ data-complexity-scoring-last-fingerprint "stale"]
+ (try
+ (data-complexity-score/record-score! "republish-fp" "appdb" (stub-result false))
+ (let [recompute? (atom false)
+ handed (atom nil)]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))
+ complexity/republish-score! (fn [s]
+ (reset! handed s)
+ (with-meta s {::complexity/snowplow-published? true}))]
+ (#'task.complexity-score/republish-cached-score! "republish-fp")
+ (is (false? @recompute?) "must re-publish the cached score, never recompute")
+ (is (some? @handed) "cached snapshot handed to republish-score!")
+ (is (= "republish-fp" (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint advances on a confirmed re-publish")))
+ (finally
+ (t2/delete! :model/DataComplexityScore :fingerprint "republish-fp")))))
+ (testing "failed re-publish leaves the fingerprint stale for the next retry"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"]
+ (try
+ (data-complexity-score/record-score! "republish-fail-fp" "appdb" (stub-result false))
+ (mt/with-dynamic-fn-redefs [complexity/republish-score! (fn [s] (with-meta s {::complexity/snowplow-published? false}))]
+ (#'task.complexity-score/republish-cached-score! "republish-fail-fp")
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint preserved when the re-publish fails"))
+ (finally
+ (t2/delete! :model/DataComplexityScore :fingerprint "republish-fail-fp")))))
+ (testing "cooldown saw a row that's since gone → falls back to a full recompute"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"]
+ (let [recompute? (atom false)]
+ (mt/with-dynamic-fn-redefs [data-complexity-score/latest-score (constantly nil)
+ complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))]
+ (#'task.complexity-score/republish-cached-score! "missing-fp")
+ (is (true? @recompute?) "a missing snapshot must trigger a recompute"))))))))
-(deftest ^:sequential cron-skips-when-boot-run-holds-active-claim-test
- (testing "the cron job's scoring call skips when the boot-time run is already mid-flight for the
- current fingerprint — the shared claim protocol serializes the two paths so they can't
- both compute and publish in parallel"
+(deftest ^:sequential cron-run-claim-path-test
+ (testing "the cron path — with-scoring-claim! + run-claim!, exactly what the Quartz job body calls"
+ (mt/initialize-if-needed! :db)
(mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
- (let [current-fp (#'task.complexity-score/current-fingerprint)
- boot-claim (pr-str {:fingerprint current-fp
- :claimed-at (#'task.complexity-score/now-ms)
- :owner "boot-path-owner-token"})
- scoring-ran? (atom false)]
- (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
- data-complexity-scoring-last-fingerprint "stale"
- data-complexity-scoring-claim boot-claim]
- (mt/with-dynamic-fn-redefs [complexity/complexity-scores
- (fn [& _] (reset! scoring-ran? true) (stub-result true))]
- ;; Exercise the cron's code path via the shared helper — this is exactly what the
- ;; Quartz job body calls. Re-sampling current-fingerprint does not matter because the
- ;; live value matches the one the boot path claimed.
- (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-scoring!)
- (is (false? @scoring-ran?)
- "cron tick skipped because the boot run holds the scoring claim")
- (is (= boot-claim (settings/data-complexity-scoring-claim))
- "boot's claim preserved — cron never took it, so it doesn't clear it")))))))
+ (testing "skips when a boot-time run already holds an active claim for the current fingerprint"
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ boot-claim (pr-str {:fingerprint current-fp
+ :claimed-at (#'task.complexity-score/now-ms)
+ :owner "boot-path-owner-token"})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim boot-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-claim!)
+ (is (false? @scoring-ran?)
+ "cron tick skipped because the boot run holds the scoring claim")
+ (is (= boot-claim (settings/data-complexity-scoring-claim))
+ "boot's claim preserved — cron never took it, so it doesn't clear it")))))
+ (testing "regression: a scored-but-unpublished snapshot within the cooldown re-publishes instead of skipping for a week"
+ (let [current (#'task.complexity-score/current-fingerprint)
+ recompute? (atom false)
+ republished? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ ;; publish lagged — last-fp never advanced to the scored fingerprint
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (try
+ (data-complexity-score/record-score! current "appdb" (stub-result false))
+ (mt/with-dynamic-fn-redefs [data-complexity-score/scored-within-cooldown? (constantly true)
+ complexity/complexity-scores (fn [& _] (reset! recompute? true) (stub-result true))
+ complexity/republish-score! (fn [s]
+ (reset! republished? true)
+ (with-meta s {::complexity/snowplow-published? true}))]
+ (#'task.complexity-score/with-scoring-claim! {:cooldown-hours 12} #'task.complexity-score/run-claim!)
+ (is (true? @republished?) "re-published the cached snapshot")
+ (is (false? @recompute?) "did not recompute")
+ (is (= current (settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint advanced after the re-publish — telemetry no longer stalled for a week"))
+ ;; The row is written under the *real* current-fingerprint with created_at=now — delete it
+ ;; so a later :sequential test exercising the real cooldown path doesn't see a spurious row.
+ (finally
+ (t2/delete! :model/DataComplexityScore :fingerprint current :source "appdb")))))))))
-(deftest ^:sequential complexity-scores-tags-publish-success-on-result-test
- (testing "complexity-scores stamps publish success/failure via metadata for schedule/boot callers"
- (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
- (constantly {:library [] :universe [] :metabot []})]
- (testing "successful publish → ::snowplow-published? true"
+(deftest ^:sequential publish-tagging-test
+ (testing "::snowplow-published? metadata records publish success/failure for schedule/boot callers"
+ (testing "complexity-scores (fresh compute + publish)"
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [] :universe [] :metabot []})]
+ (testing "successful publish → ::snowplow-published? true"
+ (snowplow-test/with-fake-snowplow-collector
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result)))))))
+ (testing "publish throw → ::snowplow-published? false (and the result is still returned)"
+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result)))))))
+ (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"
+ ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would
+ ;; look indistinguishable from a successful publish and the fingerprint would advance,
+ ;; silently suppressing the boot-time retry once tracking is turned back on.
+ (mt/with-temporary-setting-values [anon-tracking-enabled false]
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result)))))))))
+ (testing "republish-score! (re-emit cached, no recompute) mirrors that tagging"
+ (let [score {:library {:score 0 :components {}}
+ :universe {:score 0 :components {}}
+ :metabot {:score 0 :components {}}
+ :meta {:formula-version 1 :synonym-threshold 0.8 :weights {}}}]
+ (testing "publish succeeds → tagged true"
+ (snowplow-test/with-fake-snowplow-collector
+ (is (true? (::complexity/snowplow-published? (meta (complexity/republish-score! score)))))))
+ (testing "publish throws → tagged false, score still returned unchanged"
+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
+ (let [result (complexity/republish-score! score)]
+ (is (false? (::complexity/snowplow-published? (meta result))))
+ (is (= score result) "the score value is returned untouched"))))))))
+
+(deftest ^:sequential republish-from-cached-snapshot-round-trip-test
+ (testing "the real latest-score → republish-score! → emit-snowplow! round-trip emits events from a cached snapshot"
+ (mt/initialize-if-needed! :db)
+ (let [fingerprint "republish-round-trip-test/fp"
+ ;; :meta carries keyword values (:text-variant and nested :embedding-model) that JSON storage
+ ;; round-trips back as strings — the fragile bit emit-snowplow! must keep tolerating, and the
+ ;; whole point of the republish-from-cache path this commit adds.
+ score {:library {:score 1 :components {}}
+ :universe {:score 2 :components {}}
+ :metabot {:score 3 :components {}}
+ :meta {:formula-version 1
+ :synonym-threshold 0.8
+ :weights {}
+ :text-variant :names-split
+ :embedding-model {:provider :ai-service :model-name "minilm" :model-dimensions 384}}}]
+ (try
+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint)
+ (data-complexity-score/record-score! fingerprint "appdb" score)
(snowplow-test/with-fake-snowplow-collector
- (let [result (complexity/complexity-scores :embedder nil)]
- (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
- (meta result)))))))
- (testing "publish throw → ::snowplow-published? false (and the result is still returned)"
- (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
- (let [result (complexity/complexity-scores :embedder nil)]
- (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
- (meta result)))))))
- (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"
- ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would
- ;; look indistinguishable from a successful publish and the fingerprint would advance,
- ;; silently suppressing the boot-time retry once tracking is turned back on.
- (mt/with-temporary-setting-values [anon-tracking-enabled false]
- (let [result (complexity/complexity-scores :embedder nil)]
- (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
- (meta result))))))))))
+ (snowplow-test/pop-event-data-and-user-id!) ; drain startup/setting events
+ (let [cached (data-complexity-score/latest-score fingerprint "appdb")
+ result (complexity/republish-score! cached)
+ events (complexity-events!)]
+ (is (true? (::complexity/snowplow-published? (meta result)))
+ "publish reported success")
+ (is (= 3 (count events))
+ "library, universe, and metabot totals re-emitted from the cached snapshot")
+ (is (= "names_split" (get-in (first events) ["parameters" "text_variant"]))
+ "keyword :meta values survive the JSON round-trip into the emitted parameters")
+ ;; Round-tripped :embedding-model flows through the recursive snake-keys walk. Its
+ ;; `:provider` value comes back as the string "ai-service", so it stays "ai-service" — a
+ ;; fresh in-memory `:ai-service` keyword would instead snake to "ai_service".
+ (is (= {"provider" "ai-service" "model_name" "minilm" "model_dimensions" 384}
+ (get-in (first events) ["parameters" "embedding_model"]))
+ "nested :embedding-model survives the recursive snake-keys walk over round-tripped data")))
+ (finally
+ (t2/delete! :model/DataComplexityScore :fingerprint fingerprint))))))