Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Data Complexity Score (#72681)

Chris Truterd51c5d501290
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f0b8dc31e9d..30ada1b6b54 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1192,6 +1192,7 @@
{:team "Metabot"
:api #{metabase.metabot.api
metabase.metabot.api.entity-analysis
+ metabase.metabot.config
metabase.metabot.core
metabase.metabot.init
metabase.metabot.self
@@ -1237,7 +1238,7 @@
transforms-base
util
warehouses}
- :model-exports #{:model/AiUsageLog :model/MetabotMessage}
+ :model-exports #{:model/AiUsageLog :model/Metabot :model/MetabotMessage}
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
@@ -2814,6 +2815,31 @@
:api #{}
:uses #{premium-features}}
+ enterprise/data-complexity-score
+ {:team "Metabot"
+ :api #{metabase-enterprise.data-complexity-score.api
+ metabase-enterprise.data-complexity-score.init}
+ :uses #{analytics
+ api
+ app-db
+ audit-app
+ collections
+ config
+ metabot
+ premium-features
+ settings
+ startup
+ task
+ util
+ enterprise/semantic-search}
+ :model-imports #{:model/Card
+ :model/Collection
+ :model/Database
+ :model/Measure
+ :model/Metabot
+ :model/ModerationReview
+ :model/Table}}
+
enterprise/data-studio
{:team "UX West"
:api #{metabase-enterprise.data-studio.api}
@@ -3177,6 +3203,7 @@
config
connection-pool
llm
+ metabot
models
permissions
premium-features
diff --git a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj b/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj
index 6b7e5d7aea6..28bca0d05fd 100644
--- a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj
+++ b/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj
@@ -15,6 +15,7 @@
[metabase-enterprise.cloud-proxy.api]
[metabase-enterprise.content-translation.routes]
[metabase-enterprise.content-verification.api.routes]
+ [metabase-enterprise.data-complexity-score.api]
[metabase-enterprise.data-studio.api]
[metabase-enterprise.database-replication.api :as database-replication.api]
[metabase-enterprise.database-routing.api]
@@ -99,6 +100,9 @@
"/content-translation" (premium-handler metabase-enterprise.content-translation.routes/routes :content-translation)
"/cloud-add-ons" metabase-enterprise.cloud-add-ons.api/routes
"/cloud-proxy" metabase-enterprise.cloud-proxy.api/routes
+ ;; No premium-handler gate yet — we haven't settled on the feature flag name or final API shape.
+ ;; Endpoint is superuser-only so it's not exposed to regular users in the meantime.
+ "/data-complexity-score" metabase-enterprise.data-complexity-score.api/routes
"/data-studio" (premium-handler metabase-enterprise.data-studio.api/routes :library)
"/database-replication" (-> database-replication.api/routes ;; database-replication requires all these features.
(premium-handler :attached-dwh)
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index 798fc2c6574..6e34503ffc0 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -10,6 +10,7 @@
[metabase-enterprise.advanced-config.init]
[metabase-enterprise.audit-app.init]
[metabase-enterprise.cache.init]
+ [metabase-enterprise.data-complexity-score.init]
[metabase-enterprise.database-replication.init]
[metabase-enterprise.dependencies.init]
[metabase-enterprise.gsheets.init]
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
new file mode 100644
index 00000000000..8ca1d5dbeb4
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
@@ -0,0 +1,84 @@
+(ns metabase-enterprise.data-complexity-score.api
+ "Admin-only HTTP endpoint exposing the Data Complexity Score."
+ (:require
+ [metabase-enterprise.data-complexity-score.complexity :as complexity]
+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase.api.common :as api]
+ [metabase.api.macros :as api.macros]
+ [metabase.api.routes.common :refer [+auth]]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private SubScore
+ "Either a computed sub-score (`:measurement` + `:score`) or an uncomputed one (`:error`)."
+ [:or
+ [:map {:closed true}
+ [:measurement number?]
+ [:score nat-int?]]
+ [:map {:closed true}
+ [:measurement nil?]
+ [:score nil?]
+ [:error string?]]])
+
+(def ^:private Catalog
+ "One catalog's total + per-component breakdown.
+ `:total` is nil when any sub-score couldn't be computed — failures cascade through aggregates."
+ [:map
+ [:total [:maybe nat-int?]]
+ [:components
+ [:map
+ [:entity-count SubScore]
+ [:name-collisions SubScore]
+ [:synonym-pairs SubScore]
+ [:field-count SubScore]
+ [:repeated-measures SubScore]]]])
+
+(def ^:private EmbeddingModelMeta
+ "Identifies the embedding model backing the synonym calculations, so benchmark consumers can pin to it.
+ `nil` when semantic search isn't configured on this instance."
+ [:maybe
+ [:map
+ [:provider string?]
+ [:model-name string?]]])
+
+(def ^:private ComplexityScoresResponse
+ "Full response body for `GET /api/ee/data-complexity-score/complexity`."
+ [:map
+ [:library Catalog]
+ [:universe Catalog]
+ [:metabot Catalog]
+ [:meta
+ [:map
+ [:formula-version pos-int?]
+ [:synonym-threshold number?]
+ [:embedding-model {:optional true} EmbeddingModelMeta]]]])
+
+;; Per-JVM single-flight guard for the /complexity endpoint. Each scoring run walks the entire
+;; app-db catalog and emits Snowplow events, so concurrent superuser requests on the same node
+;; would just multiply load and noise without producing different results — fast-fail with 409
+;; instead. In a clustered deployment the guard is per-node, so up to one pass per node can still
+;; run concurrently; we accept that since superuser API traffic is low-volume. The Quartz job
+;; already has its own concurrency control (`DisallowConcurrentExecution` + cluster lock for boot
+;; emission), so we deliberately don't share this guard with the task path; a daily cron run that
+;; coincided with an API call shouldn't be cancelled.
+(defonce ^:private ^java.util.concurrent.atomic.AtomicBoolean api-scoring-running?
+ (java.util.concurrent.atomic.AtomicBoolean. false))
+
+(api.macros/defendpoint :get "/complexity" :- ComplexityScoresResponse
+ "Return the current Data Complexity Score for this instance.
+ Superuser-only, expensive, and emits Snowplow events for benchmark consumers. Concurrent
+ requests on the same JVM fast-fail with HTTP 409 — a scoring pass walks the full app-db
+ catalog and one in-flight run per node is enough. The guard is per-JVM, so in a clustered
+ deployment each node can still run one pass concurrently."
+ [_route _query _body]
+ (api/check-superuser)
+ (when-not (.compareAndSet api-scoring-running? false true)
+ (throw (ex-info "Data Complexity Score calculation already in progress" {:status-code 409})))
+ (try
+ (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))
+ (finally
+ (.set api-scoring-running? false))))
+
+(def ^{:arglists '([request respond raise])} routes
+ "`/api/ee/data-complexity-score` routes."
+ (api.macros/ns-handler *ns* +auth))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
new file mode 100644
index 00000000000..ad2cf22d317
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -0,0 +1,482 @@
+(ns metabase-enterprise.data-complexity-score.complexity
+ "Computes the Data Complexity Score for the semantic layer across three catalogs:
+ :library — the curated subset (Cards of type :model and :metric)
+ :universe — everything (library entities + all active physical tables)
+ :metabot — what the internal Metabot can surface, narrowed by a caller-supplied scope."
+ (:require
+ [clojure.pprint :as pprint]
+ [clojure.string :as str]
+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
+ [metabase-enterprise.semantic-search.core :as semantic-search]
+ [metabase.analytics.core :as analytics]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.audit-app.core :as audit]
+ [metabase.collections.core :as collections]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def formula-version
+ "Bump when the scoring formula changes in a way that would break historical comparisons."
+ 1)
+
+(def weights
+ "Per-axis weights applied to raw measurements. Public because they're part of the scoring
+ fingerprint — a tuning change must force a re-score and be visible to Snowplow consumers
+ without bumping `formula-version`."
+ {:entity 10
+ :name-collision 100
+ :synonym-pair 50
+ :field 1
+ :repeated-measure 2})
+
+(def ^:private component->group
+ "Thematic parent per sub-component — drives the `<group>.total` + `<group>.<component>` rollup in
+ emitted keys so operators can tell `size` from `ambiguity` without SQL.
+ Must cover every key produced by [[score-catalog]] or the missing ones emit as `nil.<component>`."
+ {:entity-count :size
+ :field-count :size
+ :name-collisions :ambiguity
+ :synonym-pairs :ambiguity
+ :repeated-measures :ambiguity})
+
+(def synonym-similarity-threshold
+ "Cosine-similarity cutoff for flagging two names as synonyms.
+ Higher than semantic-search's retrieval cutoff (0.30) because scoring needs precision, not recall.
+ See https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb for how this
+ value was chosen."
+ 0.90)
+
+;;; ----------------------------------- enumeration -----------------------------------
+;;;
+(defn- table-field-counts
+ "Return `{table-id field-count}` for active fields on the given `table-ids`. Single group-by query."
+ [table-ids]
+ (if (empty? table-ids)
+ {}
+ (into {}
+ (map (juxt :table_id :field_count))
+ (t2/query {:select [:table_id [:%count.* :field_count]]
+ :from [:metabase_field]
+ :where [:and
+ [:= :active true]
+ [:in :table_id table-ids]]
+ :group-by [:table_id]}))))
+
+(defn- table-measure-names
+ "Return `{table-id [measure-name ...]}` for non-archived Measures on the given `table-ids`. A
+ measure is a named MBQL aggregation attached to a Table — see [[metabase.measures.models.measure]]."
+ [table-ids]
+ (if (empty? table-ids)
+ {}
+ (u/group-by :table_id :name
+ (t2/select [:model/Measure :table_id :name]
+ :archived false
+ :table_id [:in table-ids]))))
+
+(defn- ->card-entity
+ "Shape a Card row into an entity map for scoring. Cards don't contribute to `:field-count` in
+ v1 — the proposal's +1-per-field rule is about physical Table fields, not Card result columns —
+ so we can skip the fat `result_metadata` column entirely. Measures are a separate first-class
+ model tied to Tables, not Cards, so Cards also don't contribute to `:measure-names`."
+ [{:keys [id name type]}]
+ {:id id
+ :name name
+ :kind (keyword type)
+ :field-count 0
+ :measure-names []})
+
+(defn- ->table-entity [field-counts measure-names {:keys [id name]}]
+ {:id id
+ :name name
+ :kind :table
+ :field-count (get field-counts id 0)
+ :measure-names (get measure-names id [])})
+
+(defn- library-collection-ids
+ "Set of collection IDs that make up the Library (root + descendants). Empty when the instance has no Library yet."
+ []
+ (into #{}
+ (when-let [root (collections/library-collection)]
+ ;; This is cheaper and less brittle than referencing the collection type constants for every entity type.
+ (cons (:id root) (collections/descendant-ids root)))))
+
+(defn- metabot-collection-scope-ids
+ "Set of collection IDs the internal Metabot can see — its `collection_id` plus descendants.
+ nil when no collection scope is configured (Metabot retrieves from everywhere). If the
+ collection row can't be loaded (stale/invalid id) we still return a singleton set with the
+ raw id so the catalog matches `metabot-metrics-and-models-query`, which filters on the raw
+ `collection_id` and returns an empty result rather than dropping the filter."
+ [collection-id]
+ (when collection-id
+ (into #{collection-id}
+ (when-let [root (t2/select-one :model/Collection :id collection-id)]
+ (collections/descendant-ids root)))))
+
+(defn- verified-card-id-set
+ "Set of Card ids whose most-recent moderation review is `verified`. Called only when
+ `metabot-scope` requests verified-only filtering — avoids a `moderation_review` join on the
+ universe Card select by pushing the check into a small auxiliary lookup."
+ []
+ (t2/select-fn-set :moderated_item_id :model/ModerationReview
+ :moderated_item_type "card"
+ :most_recent true
+ :status "verified"))
+
+(defn- routed-child-database-id-set
+ "Set of database ids whose `router_database_id` is non-nil — the routed child databases whose
+ tables Metabot/search hide. Tables with `:db_id` in this set are excluded from the `:metabot`
+ catalog, mirroring the table-visibility rule in `metabase.warehouse-schema.models.table`."
+ []
+ (t2/select-fn-set :id :model/Database :router_database_id [:not= nil]))
+
+(defn- pick-by-row
+ "Filter `entities` by `row-pred` applied to the correspondingly-indexed `rows`. Preserves
+ reference identity on the kept entity maps so library/metabot vectors share map instances
+ with the universe vector rather than allocating fresh ones.
+ A nil `row-pred` short-circuits to `[]` — callers whose filter is statically empty (e.g. no
+ Library collections on this instance) pass nil to skip enumeration entirely."
+ [row-pred rows entities]
+ (if row-pred
+ (into []
+ (keep-indexed (fn [i row] (when (row-pred row) (nth entities i))))
+ rows)
+ []))
+
+(defn- enumerate-catalogs
+ "One-pass enumeration of all three scoring catalogs. Returns
+ `{:library [...] :universe [...] :metabot [...]}` where entity maps are shared *by reference*
+ across catalogs — a Card or Table that appears in more than one catalog is one map in memory,
+ not three.
+
+ An earlier revision fetched the three catalogs separately, which duplicated DB work up to 3×
+ per scoring run — most expensively on [[table-field-counts]] and [[table-measure-names]],
+ each a `GROUP BY` scan over `metabase_field` / `measure`. We now fetch the universe superset
+ once and derive the `:library` and `:metabot` subsets by in-memory filter, which collapses 6
+ DB round-trips + 2 auxiliaries into 4 + up-to-2 and lets the aggregates run once each.
+
+ `metabot-scope` is `{:verified-only? <bool> :collection-id <nil|Long>}` describing how the
+ internal Metabot narrows its Cards further — it only adds filters, never widens. The caller
+ owns the scope decision (premium-feature gate + Metabot row lookup); this namespace does not
+ read settings, premium-feature gates, or Metabot rows directly."
+ [{:keys [verified-only? collection-id]}]
+ (let [library-cids (library-collection-ids)
+ metabot-cids (metabot-collection-scope-ids collection-id)
+ verified-ids (when verified-only? (verified-card-id-set))
+ routed-db-ids (routed-child-database-id-set)
+ ;; Extra columns (`:collection_id`, `:is_published`, `:visibility_type`, `:db_id`) are
+ ;; selected purely to drive the in-memory library/metabot derivations below — they're
+ ;; ignored by `->card-entity` / `->table-entity`. `:card_schema` is required by
+ ;; `:model/Card`'s post-select hooks even when we don't otherwise use it.
+ universe-cards (t2/select [:model/Card :id :name :type :collection_id :card_schema]
+ :type [:in ["metric" "model"]]
+ :archived false
+ :database_id [:not= audit/audit-db-id])
+ universe-tables (t2/select [:model/Table :id :name :collection_id :is_published
+ :visibility_type :db_id]
+ :active true
+ :db_id [:not= audit/audit-db-id])
+ field-counts (table-field-counts (mapv :id universe-tables))
+ measure-names (table-measure-names (mapv :id universe-tables))
+ card-entities (mapv ->card-entity universe-cards)
+ table-entities (mapv #(->table-entity field-counts measure-names %) universe-tables)
+ in-library-card? (when (seq library-cids)
+ (fn [{:keys [collection_id]}]
+ (contains? library-cids collection_id)))
+ in-library-table? (when (seq library-cids)
+ (fn [{:keys [collection_id is_published]}]
+ (and is_published
+ (contains? library-cids collection_id))))
+ in-metabot-card? (when (and (or (nil? metabot-cids) (seq metabot-cids))
+ (or (not verified-only?) (seq verified-ids)))
+ (fn [{:keys [collection_id id]}]
+ (and (or (nil? metabot-cids)
+ (contains? metabot-cids collection_id))
+ (or (not verified-only?)
+ (contains? verified-ids id)))))
+ in-metabot-table? (fn [{:keys [visibility_type db_id]}]
+ (and (nil? visibility_type)
+ (not (contains? routed-db-ids db_id))))]
+ {:universe (into card-entities table-entities)
+ :library (into (pick-by-row in-library-card? universe-cards card-entities)
+ (pick-by-row in-library-table? universe-tables table-entities))
+ :metabot (into (pick-by-row in-metabot-card? universe-cards card-entities)
+ (pick-by-row in-metabot-table? universe-tables table-entities))}))
+
+;;; ------------------------------------- scoring -------------------------------------
+
+(defn- component-score
+ "Sub-score map: raw `:measurement` (double — future-proofs non-integer axes like density) and
+ weighted `:score` using the weight at `weight-key`."
+ [weight-key n]
+ {:measurement (double n)
+ :score (* n (get weights weight-key))})
+
+(defn- repeated-names
+ "Count of name occurrences past the first (normalized for comparison). Single pass, no
+ intermediate frequency map. `raw-names` may contain nils — they're skipped."
+ [raw-names]
+ (second
+ (reduce (fn [[seen repeats] raw-name]
+ (if-let [n-name (embedders/normalize-name raw-name)]
+ (if (contains? seen n-name)
+ [seen (inc repeats)]
+ [(conj seen n-name) repeats])
+ [seen repeats]))
+ [#{} 0]
+ raw-names)))
+
+(defn- score-entity-count [entities]
+ (component-score :entity (count entities)))
+
+(defn- score-name-collisions [entities]
+ (component-score :name-collision (repeated-names (map :name entities))))
+
+(defn- score-field-count [entities]
+ (component-score :field (reduce + (keep :field-count entities))))
+
+(defn- score-repeated-measures [entities]
+ (component-score :repeated-measure (repeated-names (mapcat :measure-names entities))))
+
+(defn- dot ^double [^floats a ^floats b]
+ (let [len (alength a)]
+ (loop [i 0 acc 0.0]
+ (if (< i len)
+ (recur (inc i) (+ acc (* (aget a i) (aget b i))))
+ acc))))
+
+(defn- synonym-pair?
+ "True when two vectors' cosine similarity is ≥ sqrt(`threshold-sq`).
+ Uses the squared form of the inequality — `(a·b)² ≥ t² · ‖a‖² · ‖b‖²` when `a·b ≥ 0`, avoiding
+ two `Math/sqrt` calls a direct cosine-similarity computation would need.
+ The non-negative guard keeps it sound (squaring a negative `a·b` would flip the inequality).
+
+ `norms-product` is `‖a‖² · ‖b‖²` precomputed by the caller; folding it into one arg keeps us
+ within Clojure's 4-argument cap for primitive-typed `defn`s."
+ [^floats a ^floats b ^double norms-product ^double threshold-sq]
+ (and (pos? norms-product)
+ (let [dot-ab (dot a b)]
+ (and (>= dot-ab 0.0)
+ (>= (* dot-ab dot-ab) (* threshold-sq norms-product))))))
+
+(defn- synonym-pair-count
+ "Count of vector pairs whose cosine similarity is ≥ `threshold`. Walks the upper triangle of the
+ N×N pair matrix; each vector's `‖v‖²` is precomputed once and reused across every comparison it
+ participates in.
+
+ TODO: this is O(N²) in the distinct-name count. Fine while the signal source is the shared
+ search-index (bounded by what the indexer has seen), but once we introduce a dedicated name-only
+ embedder this should revisit — either as a chunked `M·Mᵀ` via Neanderthal/dtype-next or as a
+ dedicated pgvector name-index doing the join in SQL."
+ [embeddings threshold]
+ (let [n (count embeddings)
+ threshold-sq (* threshold threshold)
+ ^doubles norms-sq (double-array n)]
+ (dotimes [i n]
+ (let [v ^floats (embeddings i)]
+ (aset norms-sq ^long i (dot v v))))
+ (count
+ (for [i (range n)
+ j (range (inc ^long i) n)
+ :when (synonym-pair? (embeddings i) (embeddings j)
+ (* (aget norms-sq i) (aget norms-sq j))
+ threshold-sq)]
+ :value-doesnt-matter))))
+
+(defn- score-synonym-pairs
+ "Compute the synonym sub-score for `entities` using `embedder`. On embedder failure, returns nil
+ `:score`/`:measurement` plus an `:error` string so the failure cascades through aggregates
+ instead of being mistaken for a real zero. A nil `embedder` produces an empty lookup and scores
+ a real zero."
+ [entities embedder]
+ (try
+ (let [name->vec (or (and embedder (embedder entities)) {})
+ ;; We need to materialize all these vectors in a clojure vec for efficient pairwise similarity checks.
+ known-vectors (into []
+ (comp (keep (comp embedders/normalize-name :name))
+ (distinct)
+ (keep name->vec))
+ entities)
+ pairs (synonym-pair-count known-vectors synonym-similarity-threshold)]
+ (component-score :synonym-pair pairs))
+ (catch Throwable t
+ (log/warn t "Complexity score: synonym detection failed; cascading nil through aggregates")
+ (let [msg (some-> (.getMessage t) str/trim)
+ err (if (str/blank? msg)
+ (or (some-> (class t) .getName) "synonym detection failed")
+ msg)]
+ {:measurement nil :score nil :error err}))))
+
+(defn- nil-safe-sum
+ "Sum `xs` (numbers and/or nils). Returns nil if any element is nil — used to cascade an
+ uncomputed sub-score through aggregates instead of silently low-biasing the total with zeros."
+ [xs]
+ (when (every? some? xs)
+ (reduce + xs)))
+
+(defn score-catalog
+ "Pure: compute the score breakdown for a catalog given its `entities` and an optional `embedder`."
+ [entities embedder]
+ (let [components {:entity-count (score-entity-count entities)
+ :name-collisions (score-name-collisions entities)
+ :synonym-pairs (score-synonym-pairs entities embedder)
+ :field-count (score-field-count entities)
+ :repeated-measures (score-repeated-measures entities)}]
+ {:total (nil-safe-sum (map (comp :score val) components))
+ :components components}))
+
+;;; ----------------------------------- public API ------------------------------------
+
+(defn- log-scores!
+ "Log the result at :info so operators see the score in app logs even when Snowplow is off."
+ [result]
+ (log/info (str "Semantic complexity score:\n"
+ ;; `pprint` goes through `with-out-str`, not `*out*`, so the "use metabase.util.log" lint is n/a.
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (with-out-str (pprint/pprint result)))))
+
+(defn- snake ^String [x]
+ (str/replace (name x) "-" "_"))
+
+(defn- dotted-key
+ "Join `parts` with `.` after snake-casing each. `(dotted-key :size :entity-count)` → `\"size.entity_count\"`."
+ [& parts]
+ (str/join "." (map snake parts)))
+
+(defn- parameters-map
+ "Sorted-map of scoring inputs likely to evolve, published as a JSON object on each event.
+ String keys (top-level and nested) so they round-trip unchanged — Snowplow's `payload` only
+ snake-cases top-level keys, and Cheshire would serialize nested keyword keys with their leading
+ colon. Excludes `formula_version` — that stays top-level as the primary cross-version filter."
+ [{:keys [synonym-threshold embedding-model weights]}]
+ (cond-> (sorted-map "synonym_threshold" synonym-threshold
+ "weights" (into (sorted-map)
+ (map (fn [[k v]] [(snake k) v]))
+ weights))
+ embedding-model (assoc "embedding_model_provider" (:provider embedding-model)
+ "embedding_model_name" (:model-name embedding-model))))
+
+(def ^:private max-error-length
+ "Matches the Snowplow schema's `error` maxLength — a pathological exception message
+ must not fail validation and drop the whole event."
+ 1024)
+
+(defn- truncate-error [s]
+ (cond-> s (< max-error-length (count s)) (subs 0 max-error-length)))
+
+(defn- emit-snowplow!
+ "Submits Snowplow events for every score, every group aggregation, and the grand total.
+ Returns true when they are all successfully delivered.
+ Returns false when tracking is disabled or any emission failed."
+ [{:keys [library universe metabot meta]}]
+ (let [base {:event :data_complexity_scoring
+ :formula_version (:formula-version meta)
+ :parameters (parameters-map meta)}
+ events (for [[catalog result] [[:library library] [:universe universe] [:metabot metabot]]
+ event (concat (for [[component sub] (:components result)]
+ ;; leaf component
+ (cond-> (assoc base
+ :catalog catalog
+ :key (dotted-key (component->group component) component)
+ :score (:score sub)
+ :measurement (:measurement sub))
+ (:error sub) (assoc :error (truncate-error (:error sub)))))
+ (for [[group entries] (group-by #(component->group (key %)) (:components result))]
+ ;; group total
+ (assoc base
+ :catalog catalog
+ :key (dotted-key group :total)
+ :score (nil-safe-sum (map (comp :score val) entries))))
+ ;; grand total
+ [(assoc base
+ :catalog catalog
+ :key (dotted-key :total)
+ :score (:total result))])]
+ event)]
+ ;; No short-circuiting - even if they are failures, attempt the rest.
+ (reduce (fn [all-ok? event]
+ (and (analytics/track-event! :snowplow/data_complexity event) all-ok?))
+ ;; Since events cannot be empty, we don't need to worry about returning a vacuous true.
+ true
+ events)))
+
+(defn- time-phase!
+ "Run `f`, record duration on the per-phase histogram labelled by `stage` and `catalog`, return its value."
+ [stage catalog f]
+ (let [timer (u/start-timer)]
+ (try
+ (f)
+ (finally
+ (prometheus/observe! :metabase-data-complexity/phase-duration-ms
+ {:stage stage :catalog catalog}
+ (u/since-ms timer))))))
+
+(defn complexity-scores
+ "Compute the complexity score for the `:library`, `:universe`, and `:metabot` catalogs of this
+ Metabase instance. Returns a map of the shape:
+
+ {:library {:total n :components {...}}
+ :universe {:total n :components {...}}
+ :metabot {:total n :components {...}}
+ :meta {:formula-version 1
+ :synonym-threshold 0.90
+ :embedding-model {...}}}
+
+ Options:
+ `:embedder` — overrides the synonym-axis embedder (defaults to
+ [[metabase-enterprise.semantic-search.core/search-index-embedder]]); pass `nil` to disable
+ synonym scoring.
+ `:metabot-scope` — a `{:verified-only? <bool> :collection-id <nil|Long>}` map describing how
+ the internal Metabot filters Cards. `:metabot` is always scored separately from `:universe`
+ because Metabot/search table visibility (hidden tables, routed databases) already diverges
+ from the raw `:universe` set; the scope only narrows Cards further. The caller owns the
+ scope decision (premium-feature gate + Metabot row lookup) so this namespace stays free of
+ settings/feature/Metabot-row reads."
+ [& {:keys [embedder metabot-scope] :as opts}]
+ ;;; NOTE: we fully materialize a vector off all entities, along with one of those in the library, rather than
+ ;;; returning reducibles. For very large instances that holds a non-trivial slim-entity list in memory
+ ;;; (name, kind, field-count, measure-names — no fat columns like `result_metadata`),
+ ;;; but each catalog is consumed by FIVE sub-score functions that each walk the collection, so making this
+ ;;; reducible would re-query the app-db five times per scoring call — a worse tradeoff than the bounded memory
+ ;;; we currently will currently consume (provided we have that memory).
+ (let [total-timer (u/start-timer)]
+ (try
+ (let [embedder (if (contains? opts :embedder)
+ embedder
+ semantic-search/search-index-embedder)
+ model-meta (when (= embedder semantic-search/search-index-embedder)
+ (semantic-search/active-embedding-model))
+ {:keys [library universe metabot]}
+ ;; Single enumerate phase — see [[enumerate-catalogs]]. Library ⊆ universe and
+ ;; metabot ⊆ universe, so fetching each catalog separately duplicated DB work; the
+ ;; catalog label `"all"` on the enumerate stage reflects that one pass covers all
+ ;; three. Per-catalog timing only applies to the pure scoring step.
+ (time-phase! "enumerate" "all" #(enumerate-catalogs metabot-scope))
+ universe-score (time-phase! "score" "universe" #(score-catalog universe embedder))
+ library-score (time-phase! "score" "library" #(score-catalog library embedder))
+ metabot-score (time-phase! "score" "metabot" #(score-catalog metabot embedder))
+ result {:library library-score
+ :universe universe-score
+ :metabot metabot-score
+ :meta (cond-> {:formula-version formula-version
+ :synonym-threshold synonym-similarity-threshold}
+ model-meta (assoc :embedding-model model-meta))}]
+ (log-scores! result)
+ (let [published? (try
+ (emit-snowplow! result)
+ (catch Throwable t
+ (log/warn t "Failed to publish complexity score to Snowplow")
+ false))]
+ ;; `emit-snowplow!` returns true only when every event reached the tracker (false when
+ ;; Snowplow is disabled or any emission failed) — scheduler/boot callers gate
+ ;; `data-complexity-scoring-last-fingerprint` on this so a disabled collector or any
+ ;; partial failure doesn't silently mark the fingerprint as published.
+ (with-meta result {::snowplow-published? published?})))
+ (finally
+ (prometheus/observe! :metabase-data-complexity/scoring-duration-ms
+ (u/since-ms total-timer))))))
+
+(comment
+ (complexity-scores))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
new file mode 100644
index 00000000000..ce2cdfc04b0
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
@@ -0,0 +1,24 @@
+(ns metabase-enterprise.data-complexity-score.complexity-embedders
+ "Pluggable embedding sources for the complexity score's synonym axis.
+ An embedder takes entities `{:id :name :kind}` and returns `{normalized-name -> ^floats vector}`,
+ omitting entities without a known vector."
+ (:require
+ [clojure.string :as str]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn normalize-name
+ "Canonical form used for name-based lookups and comparisons. nil-safe."
+ [s]
+ (some-> s str/trim u/lower-case-en))
+
+(defn fn-embedder
+ "Build an embedder that delegates to a plain `(name-embed-fn names) -> [vectors]` function.
+ Distinct normalized names are passed in; the returned vectors are zipped back by position.
+ Names whose `name-embed-fn` returns nil are omitted from the result map."
+ [name-embed-fn]
+ (fn embed [entities]
+ (let [names (->> entities (keep (comp normalize-name :name)) distinct vec)
+ vectors (when (seq names) (vec (name-embed-fn names)))]
+ (into {} (filter val) (zipmap names vectors)))))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.clj
new file mode 100644
index 00000000000..2c07d18763c
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.clj
@@ -0,0 +1,14 @@
+(ns metabase-enterprise.data-complexity-score.init
+ "Loader + startup wiring for data-complexity-score tasks.
+ Ensures boot-time Data Complexity Score publishing runs regardless of `MB_DISABLE_SCHEDULER`."
+ (:require
+ [metabase-enterprise.data-complexity-score.settings]
+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
+ [metabase.startup.core :as startup]
+ [metabase.util.quick-task :as quick-task]))
+
+(set! *warn-on-reflection* true)
+
+(defmethod startup/def-startup-logic! ::EmitComplexityScoreIfStale [_]
+ ;; Background thread — keeps boot latency flat even when scoring is slow.
+ (quick-task/submit-task! task.complexity-score/maybe-emit-boot-score!))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.clj
new file mode 100644
index 00000000000..0a67d6af20c
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.clj
@@ -0,0 +1,15 @@
+(ns metabase-enterprise.data-complexity-score.metabot-scope
+ "Resolves the internal Metabot's retrieval scope for the `:metabot` catalog."
+ (:require
+ [metabase.metabot.config :as metabot.config]
+ [metabase.premium-features.core :as premium-features]
+ [toucan2.core :as t2]))
+
+(defn internal-metabot-scope
+ "`{:verified-only? <bool> :collection-id <nil|Long>}` matching what Metabot's Card query would apply."
+ []
+ (let [entity-id (get-in metabot.config/metabot-config [metabot.config/internal-metabot-id :entity-id])
+ metabot (t2/select-one :model/Metabot :entity_id entity-id)]
+ {:verified-only? (and (boolean (:use_verified_content metabot))
+ (premium-features/has-feature? :content-verification))
+ :collection-id (:collection_id metabot)}))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj
new file mode 100644
index 00000000000..e1db149d62f
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj
@@ -0,0 +1,32 @@
+(ns metabase-enterprise.data-complexity-score.settings
+ "Settings for the data-complexity-score module."
+ (:require
+ [metabase.settings.core :refer [defsetting]]
+ [metabase.util.i18n :refer [deferred-tru]]))
+
+(defsetting data-complexity-scoring-enabled
+ (deferred-tru "Run the scheduled Data Complexity Score job (one node per cluster, daily).")
+ :encryption :no
+ :visibility :admin
+ :default false
+ :type :boolean
+ :export? false
+ :doc false)
+
+(defsetting data-complexity-scoring-last-fingerprint
+ (deferred-tru "Internal bookkeeping: Fingerprint of last successful run, to prevent needless re-calculations.")
+ :encryption :no
+ :visibility :internal
+ :default ""
+ :type :string
+ :export? false
+ :doc false)
+
+(defsetting data-complexity-scoring-claim
+ (deferred-tru "Internal bookkeeping: EDN metadata with TTL for in-progress run, to skip duplicate work.")
+ :encryption :no
+ :visibility :internal
+ :default ""
+ :type :string
+ :export? false
+ :doc false)
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
new file mode 100644
index 00000000000..4f75dd4461b
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
@@ -0,0 +1,198 @@
+(ns metabase-enterprise.data-complexity-score.task.complexity-score
+ "Daily Quartz job that computes and publishes the Data Complexity Score."
+ (:require
+ [clojure.edn :as edn]
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase-enterprise.data-complexity-score.complexity :as complexity]
+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase-enterprise.data-complexity-score.settings :as settings]
+ [metabase-enterprise.semantic-search.core :as semantic-search]
+ [metabase.app-db.cluster-lock :as cluster-lock]
+ [metabase.config.core :as config]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log])
+ (:import
+ (org.quartz DisallowConcurrentExecution)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private job-key (jobs/key "metabase.task.data-complexity-score.job"))
+(def ^:private trigger-key (triggers/key "metabase.task.data-complexity-score.trigger"))
+
+(defn- current-fingerprint
+ "String capturing everything that changes the meaning of an emitted score — mirror of the Snowplow
+ `formula_version` + `parameters` fields. Includes `weights` so re-tuning forces a re-score
+ without bumping `formula-version`; only structural changes to the scoring algorithm need that."
+ []
+ (let [embedding-model (semantic-search/active-embedding-model)]
+ (pr-str (into (sorted-map)
+ (cond-> {:formula-version complexity/formula-version
+ :synonym-threshold complexity/synonym-similarity-threshold
+ :weights complexity/weights}
+ embedding-model (assoc :embedding-model embedding-model))))))
+
+(defn- run-scoring!
+ "One scoring pass. Gated by [[settings/data-complexity-scoring-enabled]] so admins can silence
+ scoring without unscheduling the job.
+
+ `claim-fingerprint` is the fingerprint carried on the scoring claim that authorized this run.
+ Using it (rather than re-sampling [[current-fingerprint]] at commit time) means a config change
+ mid-run cannot make us stamp a fingerprint onto `last-fingerprint` that we didn't actually score.
+
+ Returns the score result (with `::complexity/snowplow-published?` metadata) when scoring ran, or
+ nil when skipped / threw. Only a confirmed Snowplow publish advances `last-fingerprint` — any
+ other outcome leaves it untouched so the next boot or cron retries and telemetry doesn't
+ silently stall behind a transient publish failure."
+ [claim-fingerprint]
+ (if (settings/data-complexity-scoring-enabled)
+ (try
+ (let [result (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))]
+ (if (::complexity/snowplow-published? (meta result))
+ (settings/data-complexity-scoring-last-fingerprint! claim-fingerprint)
+ (log/warn "Data Complexity Score: Snowplow publish failed; leaving fingerprint unchanged so the next boot or cron retries"))
+ result)
+ (catch Throwable t
+ (log/warn t "Data Complexity Score run failed")))
+ (log/debug "Data Complexity Score run skipped — data-complexity-scoring-enabled is off")))
+
+;; Long enough that any realistic scoring run finishes well inside it, short enough that a crashed
+;; claimant unblocks the next tick's retry without operator intervention.
+(def ^:private scoring-claim-ttl-ms (* 30 60 1000))
+
+(defn- now-ms [] (System/currentTimeMillis))
+
+(defn- parse-scoring-claim
+ "Parse the persisted claim string into `{:fingerprint <str> :claimed-at <long> :owner <str>}` or
+ nil when the setting is empty/garbled."
+ [s]
+ (when (and s (seq s))
+ (try
+ (let [v (edn/read-string s)]
+ (when (map? v) v))
+ (catch Throwable _ nil))))
+
+(defn- scoring-claim-active?
+ "True when a persisted claim for the same fingerprint we'd score now is still within the TTL —
+ meaning another path (sibling boot, cron tick) is already computing and publishing for it, so
+ this caller should skip."
+ [claim current-fp]
+ (boolean
+ (and claim
+ (= current-fp (:fingerprint claim))
+ (when-let [ts (:claimed-at claim)]
+ (and (int? ts)
+ (< (- (now-ms) (long ts)) scoring-claim-ttl-ms))))))
+
+(defn- claim-scoring-run!
+ "Cluster-safe claim on one Data Complexity Score run — shared by the cron job and the boot-time
+ startup hook so the two paths cannot compute and publish in parallel. Returns the claim map
+ (including a unique `:owner` token) when this caller wins the right to run, or nil when another
+ node/path already holds an active claim for the current fingerprint — or, when
+ `:require-fingerprint-change?` is true, when the live fingerprint still matches the last
+ successfully-published one (boot's extra gate so restarts at an unchanged config don't re-emit).
+
+ The returned claim must be passed back to [[release-scoring-claim!]] so the release is a
+ compare-and-clear — a sibling that took over after a TTL-based expiry keeps its own claim.
+
+ We do not advance [[settings/data-complexity-scoring-last-fingerprint]] here — only `run-scoring!`
+ does, after confirmed Snowplow publish. Instead we write a separate
+ [[settings/data-complexity-scoring-claim]] marker (fingerprint + timestamp + owner) so that:
+
+ - other nodes/paths acquiring the lock next see an active claim and skip (no duplicate scoring);
+ - a scoring/publish failure or crash mid-run leaves the success fingerprint untouched, so the
+ next boot or cron retries instead of treating the fingerprint as up-to-date;
+ - the TTL on `:claimed-at` means even a hard JVM crash doesn't permanently suppress scoring;
+ - the `:owner` token lets a slow claimant distinguish its own claim from a sibling's takeover.
+
+ Reads inside the lock bypass the in-process settings cache via `config/*disable-setting-cache*`
+ so values freshly committed by sibling nodes are always visible — the cache is only refreshed
+ periodically and without bypass two nodes could both pass the check and both score."
+ [{:keys [require-fingerprint-change?]}]
+ (cluster-lock/with-cluster-lock {:lock ::complexity-score-run
+ :timeout-seconds 10}
+ (binding [config/*disable-setting-cache* true]
+ (let [current (current-fingerprint)]
+ (when (and (settings/data-complexity-scoring-enabled)
+ (or (not require-fingerprint-change?)
+ (not= current (settings/data-complexity-scoring-last-fingerprint)))
+ (not (scoring-claim-active?
+ (parse-scoring-claim (settings/data-complexity-scoring-claim))
+ current)))
+ (let [claim {:fingerprint current
+ :claimed-at (now-ms)
+ :owner (str (random-uuid))}]
+ (settings/data-complexity-scoring-claim! (pr-str claim))
+ claim))))))
+
+(defn- release-scoring-claim!
+ "Best-effort compare-and-clear of the caller's scoring claim. Only clears the persisted value
+ when its `:owner` still matches `claim` — protecting against the case where our run outlived the
+ TTL and a sibling legitimately took over the claim in the meantime. Failure is non-fatal: the TTL
+ on the claim timestamp ensures the next boot/cron can retry either way."
+ [claim]
+ (try
+ (cluster-lock/with-cluster-lock {:lock ::complexity-score-run
+ :timeout-seconds 10}
+ (binding [config/*disable-setting-cache* true]
+ (let [persisted (parse-scoring-claim (settings/data-complexity-scoring-claim))]
+ (if (and persisted (= (:owner persisted) (:owner claim)))
+ (settings/data-complexity-scoring-claim! "")
+ (log/info "Data Complexity Score: skipping claim release — persisted claim no longer belongs to this node (sibling takeover after TTL)")))))
+ (catch Throwable t
+ (log/warn t "Data Complexity Score: failed to clear scoring claim; it will expire via TTL"))))
+
+(defn- with-scoring-claim!
+ "Acquire a scoring claim, invoke `f` with the claim's fingerprint, then release the claim via
+ compare-and-clear. Returns nil when the claim couldn't be acquired (another path is scoring, or
+ the fingerprint-change gate didn't fire). `opts` pass through to [[claim-scoring-run!]]."
+ [opts f]
+ (when-let [claim (claim-scoring-run! opts)]
+ (try
+ (f (:fingerprint claim))
+ (finally
+ (release-scoring-claim! claim)))))
+
+(task/defjob ^{DisallowConcurrentExecution true
+ :doc "Compute and publish the Data Complexity Score."}
+ DataComplexityScoring [_ctx]
+ (with-scoring-claim! {} run-scoring!))
+
+(defn maybe-emit-boot-score!
+ "Run one scoring pass at boot when the persisted fingerprint lags the live config. Shares the
+ cluster-wide scoring claim with the cron job (see [[claim-scoring-run!]]) so a restart that
+ lands on top of a 03:17 UTC cron tick can't double-publish. Runs regardless of Quartz state so
+ operators still get a score on nodes with `MB_DISABLE_SCHEDULER=true` or a failed scheduler init.
+ Intended to be called from a startup hook on a background thread — see
+ `metabase-enterprise.data-complexity-score.init`.
+
+ Success advances the last-successful fingerprint inside `run-scoring!`; any other outcome (skip,
+ throw, publish failure) leaves it untouched so the next boot or cron retries. The claim is
+ released (compare-and-clear on `:owner`) after the run so other paths can proceed without
+ waiting for the TTL — unless our run outlived the TTL and another path took over the claim."
+ []
+ (try
+ (with-scoring-claim! {:require-fingerprint-change? true}
+ (fn [fp]
+ (log/info "Data Complexity Score: fingerprint changed, emitting boot-time score")
+ (run-scoring! fp)))
+ (catch Throwable t
+ (log/warn t "Data Complexity Score: boot-time emission failed"))))
+
+(defmethod task/init! ::DataComplexityScoring [_]
+ (let [job (jobs/build
+ (jobs/of-type DataComplexityScoring)
+ (jobs/store-durably)
+ (jobs/with-identity job-key)
+ (jobs/with-description "Data Complexity Score — daily telemetry"))
+ ;; 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.
+ trigger (triggers/build
+ (triggers/with-identity trigger-key)
+ (triggers/for-job job-key)
+ (triggers/with-schedule
+ (cron/schedule
+ (cron/cron-schedule "0 17 3 * * ? *")
+ (cron/in-time-zone (java.util.TimeZone/getTimeZone "UTC"))
+ (cron/with-misfire-handling-instruction-fire-and-proceed))))]
+ (task/schedule-task! job trigger)))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index d9683ca4fbe..7add9418d76 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedders]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
@@ -15,8 +16,14 @@
[metabase.tracing.core :as tracing]
[metabase.util :as u]
[metabase.util.log :as log]
+ [potemkin :as p]
[toucan2.realize :as t2.realize]))
+(p/import-vars
+ [metabase-enterprise.semantic-search.embedders
+ active-embedding-model
+ search-index-embedder])
+
(defn- fallback-engine
"Find the highest priority search engine available for fallback."
[]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj
new file mode 100644
index 00000000000..6558f6e49ca
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj
@@ -0,0 +1,174 @@
+(ns metabase-enterprise.semantic-search.embedders
+ "Embedders that expose the pgvector semantic-search index as a name → vector lookup.
+ An embedder takes entity maps `{:id :name :kind}` and returns `{normalized-name → ^floats vector}`,
+ omitting entities not in the index."
+ (:require
+ [clojure.string :as str]
+ [honey.sql :as sql]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase.metabot.core :as metabot]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private fetch-batch-size
+ "Maximum number of `(model, model_id)` pairs per SQL query when reading from the pgvector index.
+ Keeps the generated OR predicate bounded so we don't hit JDBC bind-parameter limits or produce query
+ plans that choke on large installs."
+ 500)
+
+(def ^:private float-array-class
+ "Cached `[F` class so [[parse-pgvector]] doesn't pay a `Class/forName` lookup per row."
+ (Class/forName "[F"))
+
+(defn- normalize-name [s]
+ (some-> s str/trim u/lower-case-en))
+
+(defn- prefer-new-row?
+ "Return true when `new-row` should replace `prior` as the representative for a normalized name.
+ `search-index-embedder` keeps exactly one embedding per normalized name, so the choice of winner
+ is load-bearing: different rows carry different indexed text (name + description + other search
+ fields) and therefore different embeddings, which changes downstream similarity comparisons.
+ Priority chain (lowest wins, must be total so the result is stable across DB row order and batch
+ boundaries):
+ 1. Numeric `:mid` (parsed `model_id`) — a parseable numeric id always beats a nil-parsed one;
+ two numerics compare naturally.
+ 2. Raw `model_id` string — lexicographic; needed so `\"2\"` vs `\"02\"` (which parse equal) and
+ two non-numeric ids still order deterministically.
+ 3. `:model` string — lexicographic final tie-break when `model_id` values are identical across
+ different model types."
+ [{new-mid :mid new-model :model new-mid-str :model_id}
+ {prior-mid :mid prior-model :model prior-mid-str :model_id}]
+ (let [num-cmp (cond
+ (and new-mid prior-mid) (compare (long new-mid) (long prior-mid))
+ new-mid -1 ; numeric beats non-numeric
+ prior-mid 1
+ :else 0)
+ cmp (if-not (zero? num-cmp)
+ num-cmp
+ (let [str-cmp (compare (str new-mid-str) (str prior-mid-str))]
+ (if-not (zero? str-cmp)
+ str-cmp
+ (compare new-model prior-model))))]
+ (neg? cmp)))
+
+(defn- parse-pgvector
+ "Parse a pgvector string (\"[0.1, 0.2, ...]\") or similar into a float-array."
+ [v]
+ (cond
+ (nil? v) nil
+ (instance? float-array-class v) v
+ :else
+ (let [s (if (string? v) v (str v))
+ nums (->> (-> s
+ (str/replace "[" "")
+ (str/replace "]" "")
+ (str/split #","))
+ (keep #(let [t (str/trim %)] (when (seq t) (Float/parseFloat t)))))]
+ (float-array nums))))
+
+(defn- fetch-batch
+ "Query the active pgvector index table for a single batch of `(model, model_id)` pairs. Returns a
+ seq of `{:model :model_id :name :embedding}` rows."
+ [pgvector table-name pairs]
+ (let [where (into [:or]
+ (for [[m mid] pairs]
+ [:and [:= :model m] [:= :model_id mid]]))
+ sql-vec (sql/format {:select [:model :model_id :name :embedding]
+ :from [(keyword table-name)]
+ :where where}
+ {:quoted true})]
+ (jdbc/execute! pgvector sql-vec {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
+(defn- reduce-batched-rows
+ "Fold `rf` over rows fetched from the pgvector index in batches of [[fetch-batch-size]] pairs.
+ Never materializes the full row list — each batch is fetched, reduced into `acc`, and discarded
+ before the next batch is fetched. Keeps peak memory to `O(batch + reduced-state)` instead of
+ `O(total-rows)`, which matters because raw pgvector embedding strings are ~6–18KB each."
+ [pgvector table-name pairs rf init]
+ (reduce (fn [acc batch]
+ (reduce rf acc (fetch-batch pgvector table-name batch)))
+ init
+ (partition-all fetch-batch-size pairs)))
+
+(defn- try-active-index-state
+ "Return the active index state, or nil if unavailable. Never throws."
+ []
+ (try
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ md (semantic.env/get-index-metadata)]
+ (when-let [state (semantic.index-metadata/get-active-index-state pgvector md)]
+ {:pgvector pgvector
+ :table-name (-> state :index :table-name)
+ :model (-> state :index :embedding-model)}))
+ (catch Throwable t
+ (log/debug t "Semantic-search index not available; search-index embedder will return {}"))))
+
+(defn search-index-embedder
+ "Embedder that reads vectors from the active semantic-search pgvector index. The indexed text
+ combines name + description + other search fields, so the similarity signal reflects semantic
+ overlap rather than name-only synonymy.
+ Returns `{}` when the index isn't available (premium feature off, not yet initialized, datasource
+ unreachable).
+
+ Pgvector read failures (e.g. the index table exists but the query errors out) propagate so the
+ caller can annotate its own error state — silently returning `{}` would look indistinguishable
+ from \"no synonym matches\", which would let transient failures underreport complexity with no
+ machine-readable signal. See [[metabase-enterprise.data-complexity-score.complexity/score-synonym-pairs]]
+ for the wrapping `catch` that converts this into a `:error` field on the sub-score.
+
+ When multiple entities share a normalized name but have different indexed embeddings the row
+ chosen by [[prefer-new-row?]] wins, so the result is deterministic across runs regardless of
+ batch boundaries.
+
+ Parses embeddings as we fold so raw pgvector strings (6–18 KB each) aren't retained alongside
+ the parsed float-arrays. Peak memory is still O(distinct-names) in float-arrays, which will
+ matter on very large instances (>~50k entities) — the pairwise comparison downstream is the
+ harder scaling problem and both will want to be revisited together (push similarity into
+ pgvector SQL, or switch to HNSW approximate-neighbor)."
+ [entities]
+ (if-let [{:keys [pgvector table-name]} (try-active-index-state)]
+ ;; No explicit `distinct` on `pairs`: callers pass entities from per-catalog SQL queries where
+ ;; each `(kind, id)` maps to a unique primary key, so duplicates don't arise in practice. The
+ ;; downstream `prefer-new-row?` fold already collapses any that slip through, so the only cost
+ ;; of a hypothetical duplicate would be a slightly wider SQL `OR` clause.
+ (let [pairs (for [{:keys [id kind]} entities
+ :let [m (metabot/entity-type->search-model kind)]
+ :when m]
+ [m (str id)])
+ ;; Fold fetch → parse → dedup in one pass. `prefer-new-row?` keeps the winner globally
+ ;; (lowest numeric model_id, then model as tie-break) so the result is stable across
+ ;; runs regardless of which 500-row batch boundary a duplicate lands on.
+ keyed (when (seq pairs)
+ (reduce-batched-rows
+ pgvector table-name pairs
+ (fn [acc {:keys [name model_id model embedding]}]
+ (if-let [vec (parse-pgvector embedding)]
+ (let [k (normalize-name name)
+ entry {:mid (parse-long model_id)
+ :model model
+ :model_id model_id
+ :vec vec}
+ prior (get acc k)]
+ (if (or (nil? prior) (prefer-new-row? entry prior))
+ (assoc acc k entry)
+ acc))
+ acc))
+ {}))]
+ (update-vals (or keyed {}) :vec))
+ {}))
+
+(defn active-embedding-model
+ "Return the embedding model metadata for the *active* search index, not the current configuration.
+ Returns nil when the index is unreachable, not yet initialized, or the feature is disabled.
+ Use this instead of [[metabase-enterprise.semantic-search.env/get-configured-embedding-model]] when
+ you need to know what model the index is *actually* serving — not just what the settings say."
+ []
+ (when-let [{:keys [model]} (try-active-index-state)]
+ (when model
+ {:provider (:provider model)
+ :model-name (:model-name model)})))
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj
new file mode 100644
index 00000000000..68f5b673bb2
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj
@@ -0,0 +1,232 @@
+(ns metabase-enterprise.data-complexity-score.api-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-complexity-score.api :as api]
+ [metabase-enterprise.data-complexity-score.complexity :as complexity]
+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase.metabot.config :as metabot.config]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(comment api/keep-me)
+
+(def ^:private endpoint "ee/data-complexity-score/complexity")
+
+(defn- internal-metabot-id
+ "Primary key of the internal Metabot row — used by the tests that temporarily tweak its
+ `use_verified_content`/`collection_id` via `mt/with-temp-vals-in-db`. Calls
+ `mt/initialize-if-needed!` so the row is populated by migrations even when this test runs in
+ isolation (the endpoint tests piggyback on the web-server init and miss this failure mode)."
+ []
+ (mt/initialize-if-needed! :db)
+ (t2/select-one-pk :model/Metabot
+ :entity_id (get-in metabot.config/metabot-config
+ [metabot.config/internal-metabot-id :entity-id])))
+
+(deftest complexity-endpoint-requires-superuser-test
+ (testing "non-superusers are rejected"
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :get 403 endpoint)))))
+
+(deftest complexity-endpoint-superuser-gets-consistent-totals-test
+ (testing "check invariants not covered by schema"
+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)
+ measurement (fn [cat k] (get-in resp [cat :components k :measurement]))
+ component-score (fn [cat k] (get-in resp [cat :components k :score]))
+ ;; NOTE: `:synonym-pairs` is intentionally included here even though it's *theoretically*
+ ;; non-monotonic — `score-synonym-pairs` dedupes by normalized name and keeps whichever
+ ;; embedding `search-index-embedder` happens to pick for that name, so adding
+ ;; universe-only entities that collide on normalized name with a library entity could in
+ ;; principle flip which vector wins and drop the pair count below library's. Reviewers
+ ;; (human or AI) sometimes want to carve it out on that basis — don't. In every realistic
+ ;; configuration (prod, dev, the fixture that backs this endpoint's hermetic path in
+ ;; complexity_test.clj) the invariant holds, and asserting it keeps us honest about
+ ;; regressions in the common case. If the edge case ever actually trips this, *that* is
+ ;; the surprising thing we want to see and we'll deal with it then.
+ component-keys [:entity-count :name-collisions :synonym-pairs
+ :field-count :repeated-measures]]
+ (testing ":total equals the sum of its component :score values"
+ (doseq [catalog [:library :universe :metabot]
+ :let [{:keys [total components]} (get resp catalog)]]
+ (is (= total (reduce + (map :score (vals components))))
+ (format "%s :total should equal sum of component :score values" catalog))))
+ (testing "universe is a superset of library: every measurement and score ≥ library's"
+ (doseq [k component-keys
+ getter [measurement component-score]
+ :let [lib (getter :library k)
+ uni (getter :universe k)]]
+ (is (>= uni lib)
+ (format "universe %s (%s) should be ≥ library's (%s)" k uni lib))))
+ (testing ":synonym-pairs can't exceed the number of distinct-name pairs possible"
+ (doseq [catalog [:library :universe :metabot]
+ :let [n-entities (measurement catalog :entity-count)
+ syn-pairs (measurement catalog :synonym-pairs)
+ max-pairs (/ (* n-entities (dec n-entities)) 2)]]
+ (is (<= syn-pairs max-pairs)
+ (format "%s :synonym-pairs (%s) can't exceed n*(n-1)/2 for n=%s" catalog syn-pairs n-entities)))))))
+
+(deftest complexity-endpoint-metabot-catalog-test
+ (testing ":metabot mirrors :universe when neither content-verification nor use_verified_content is active"
+ ;; Pin both gates explicitly instead of relying on test-env defaults — the reused-verbatim path
+ ;; is only exercised when the scope is empty, and we want this assertion to keep passing even
+ ;; if the ambient defaults shift.
+ (mt/with-premium-features #{}
+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)
+ {:use_verified_content false :collection_id nil}
+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]
+ (is (= (:universe resp) (:metabot resp)))))))
+ (testing ":metabot is scored separately when :content-verification + use_verified_content are both active"
+ ;; Positive path: verified-only filtering restricts Cards to those with an active verified
+ ;; moderation review. We inject a fresh unverified Card so the assertion doesn't depend on
+ ;; ambient test-env content — the `:universe` count includes this Card, `:metabot` excludes it.
+ (mt/with-premium-features #{:content-verification}
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Card _ {:database_id db-id
+ :type :model
+ :name "Unverified Only Card"
+ :archived false}]
+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)
+ {:use_verified_content true :collection_id nil}
+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]
+ (is (< (get-in resp [:metabot :components :entity-count :measurement])
+ (get-in resp [:universe :components :entity-count :measurement]))
+ ":metabot entity-count must be strictly < :universe when verified-only filters out the injected Card")))))))
+
+(deftest complexity-endpoint-metabot-collection-scope-test
+ (testing ":metabot is scoped to the internal Metabot's collection_id subtree (root + descendants)"
+ ;; Fixture shape — exercises both halves of `metabot-collection-scope-ids`:
+ ;; parent ← Metabot's collection_id; holds a Card directly (catches root-omitted regressions)
+ ;; └ child → holds a nested Card (catches descent-dropped regressions)
+ ;; sibling → holds the out-of-subtree Card
+ ;; empty → no Cards; baseline scope so we can pin *exact* card-count deltas
+ ;;
+ ;; Tables pass through :metabot unfiltered, so ambient table counts cancel when we take
+ ;; differentials against `empty`. That leaves us a clean count of Cards visible under each
+ ;; scope, which lets us assert exact expected counts rather than relative inequalities.
+ ;; Pin premium features off explicitly so `:verified-only?` can't drift in and confound.
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Collection {parent-id :id} {:name "Metabot Scope Parent"
+ :location "/"}
+ :model/Collection {child-id :id} {:name "Metabot Scope Child"
+ :location (format "/%d/" parent-id)}
+ :model/Collection {sibling-id :id} {:name "Metabot Scope Sibling"
+ :location "/"}
+ :model/Collection {empty-id :id} {:name "Metabot Scope Empty"
+ :location "/"}
+ :model/Database {db-id :id} {}
+ :model/Card _ {:database_id db-id
+ :type :model
+ :name "Metabot Root Card"
+ :archived false
+ :collection_id parent-id}
+ :model/Card _ {:database_id db-id
+ :type :model
+ :name "Metabot In-subtree Nested Card"
+ :archived false
+ :collection_id child-id}
+ :model/Card _ {:database_id db-id
+ :type :metric
+ :name "Metabot Out-of-subtree Card"
+ :archived false
+ :collection_id sibling-id}]
+ (let [counts-with-scope (fn [cid]
+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)
+ {:use_verified_content false
+ :collection_id cid}
+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]
+ {:metabot (get-in resp [:metabot :components :entity-count :measurement])
+ :universe (get-in resp [:universe :components :entity-count :measurement])})))
+ empty-counts (counts-with-scope empty-id)
+ parent-counts (counts-with-scope parent-id)
+ sibling-counts (counts-with-scope sibling-id)
+ empty-count (:metabot empty-counts)
+ parent-count (:metabot parent-counts)
+ sibling-count (:metabot sibling-counts)]
+ (testing "scope=parent counts both the root Card and the nested descendant Card"
+ ;; If the root collection id were dropped from `metabot-collection-scope-ids`, the
+ ;; root Card would vanish and `parent-count` would land at `empty-count + 1`. If
+ ;; descent regressed, the nested Card would vanish and `parent-count` would again
+ ;; be `empty-count + 1`. Either regression fails this assertion.
+ (is (= (+ empty-count 2) parent-count)
+ "scope=parent :metabot must include BOTH parent-card (root) AND child-card (descendant)"))
+ (testing "scope=sibling counts only the in-scope Card (out-of-subtree Card is excluded)"
+ ;; If the collection-id filter were dropped entirely, `sibling-count` would jump to
+ ;; `empty-count + 3` (all three fixture Cards visible), failing this assertion.
+ (is (= (inc empty-count) sibling-count)
+ "scope=sibling :metabot must include only sibling-card — collection filter must apply"))
+ (testing ":universe entity-count is unaffected by Metabot collection scope"
+ ;; Guards against a regression where the subtree filter leaks into `:universe`
+ ;; scoring. If it did, `:universe` counts would move in lockstep with `:metabot`
+ ;; across the three scopes and this assertion would fail.
+ (is (= (:universe empty-counts)
+ (:universe parent-counts)
+ (:universe sibling-counts))
+ ":universe must be unscoped regardless of Metabot.collection_id")))))))
+
+(def ^:private empty-catalog
+ {:total 0 :components {:entity-count {:measurement 0.0 :score 0}
+ :name-collisions {:measurement 0.0 :score 0}
+ :synonym-pairs {:measurement 0.0 :score 0}
+ :field-count {:measurement 0.0 :score 0}
+ :repeated-measures {:measurement 0.0 :score 0}}})
+
+(def ^:private stub-scores
+ {:library empty-catalog :universe empty-catalog :metabot empty-catalog
+ :meta {:formula-version 1 :synonym-threshold 0.0}})
+
+(deftest ^:sequential complexity-endpoint-rejects-concurrent-requests-test
+ (testing "a second concurrent request fast-fails with 409 instead of running a duplicate scoring pass"
+ ;; Block the stubbed scoring call on a latch so the second request is guaranteed to land
+ ;; while the guard is held. Plain `with-redefs` (not `with-dynamic-fn-redefs`) because
+ ;; the dynamic variant binds thread-locally and the futures we spawn here wouldn't see it.
+ (let [release-scoring (CountDownLatch. 1)
+ scoring-started (CountDownLatch. 1)
+ call-count (atom 0)]
+ (with-redefs [complexity/complexity-scores
+ (fn [& _]
+ (swap! call-count inc)
+ (.countDown scoring-started)
+ (.await release-scoring 10 TimeUnit/SECONDS)
+ stub-scores)]
+ (let [first-request (future (mt/user-http-request :crowberto :get 200 endpoint))]
+ (try
+ (is (.await scoring-started 10 TimeUnit/SECONDS)
+ "first request must reach the guarded section before we fire the second")
+ (testing "concurrent superuser request is rejected with 409"
+ (is (= "Data Complexity Score calculation already in progress"
+ (mt/user-http-request :crowberto :get 409 endpoint))))
+ (finally
+ (.countDown release-scoring)
+ ;; Drain the in-flight request so the guard is released before the next test
+ ;; (failed assertions above shouldn't leak a stuck scoring call into other tests).
+ (deref first-request 10000 ::timeout))))
+ (testing "only the first request actually ran scoring; the second short-circuited"
+ (is (= 1 @call-count)))
+ (testing "guard is released after the in-flight request finishes — a follow-up request succeeds"
+ (mt/user-http-request :crowberto :get 200 endpoint)
+ (is (= 2 @call-count)))))))
+
+(deftest internal-metabot-scope-test
+ (testing ":verified-only? is true only when the premium feature + use_verified_content both apply"
+ (doseq [{:keys [features use-verified? expected-verified?]}
+ [{:features #{} :use-verified? false :expected-verified? false}
+ {:features #{} :use-verified? true :expected-verified? false}
+ {:features #{:content-verification} :use-verified? false :expected-verified? false}
+ {:features #{:content-verification} :use-verified? true :expected-verified? true}]]
+ (testing (format "features=%s use_verified_content=%s" (pr-str features) use-verified?)
+ (mt/with-premium-features features
+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)
+ {:use_verified_content use-verified? :collection_id nil}
+ (is (= {:verified-only? expected-verified? :collection-id nil}
+ (metabot-scope/internal-metabot-scope))))))))
+ (testing ":collection-id is read straight from the internal Metabot row regardless of premium features"
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "metabot scope test coll"}]
+ (mt/with-premium-features #{}
+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)
+ {:use_verified_content false :collection_id coll-id}
+ (is (= {:verified-only? false :collection-id coll-id}
+ (metabot-scope/internal-metabot-scope))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
new file mode 100644
index 00000000000..54aa3d57a20
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -0,0 +1,1033 @@
+(ns metabase-enterprise.data-complexity-score.complexity-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-complexity-score.complexity :as complexity]
+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
+ ;; Load init for side-effect: exercises the transitive require of the task ns so
+ ;; `scoring-task-registered-test` verifies init.clj's actual wiring path.
+ [metabase-enterprise.data-complexity-score.init]
+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase-enterprise.data-complexity-score.settings :as data-complexity-score.settings]
+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
+ [metabase-enterprise.semantic-search.core :as semantic-search]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.embedders :as ss.embedders]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.analytics.core :as analytics]
+ [metabase.analytics.snowplow-test :as snowplow-test]
+ [metabase.audit-app.core :as audit]
+ [metabase.collections.core :as collections]
+ [metabase.collections.test-utils :as collections.tu]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private test-entity-ids (atom 0))
+
+(defn- entity
+ "Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`
+ so every test entity is distinct (and so test failures print stable, readable ids)."
+ [& {:keys [name kind field-count measure-names]
+ :or {kind :table field-count 0 measure-names []}}]
+ {:id (swap! test-entity-ids inc)
+ :name name
+ :kind kind
+ :field-count field-count
+ :measure-names measure-names})
+
+(defn- mock-embedder
+ "Build an embedder backed by a `{name -> vector-literal}` lookup table for tests."
+ [name->vec-literal]
+ (embedders/fn-embedder
+ (fn [names]
+ (mapv #(when-let [v (get name->vec-literal %)] (float-array v)) names))))
+
+(deftest ^:parallel score-catalog-pure-test
+ (testing "empty catalog scores zero"
+ (is (=? {:total 0
+ :components {:entity-count {:measurement 0.0 :score 0}
+ :name-collisions {:measurement 0.0 :score 0}
+ :synonym-pairs {:measurement 0.0 :score 0}
+ :field-count {:measurement 0.0 :score 0}
+ :repeated-measures {:measurement 0.0 :score 0}}}
+ (#'complexity/score-catalog [] nil))))
+
+ (testing "entity count contributes +10 per entity"
+ (let [es [(entity :name "orders")
+ (entity :name "customers")
+ (entity :name "products")]]
+ (is (=? {:total 30
+ :components {:entity-count {:measurement 3.0 :score 30}}}
+ (#'complexity/score-catalog es nil)))))
+
+ (testing "name collisions stack linearly: 3 identical names = +200"
+ (let [es [(entity :name "orders")
+ (entity :name "orders")
+ (entity :name "orders")]]
+ (is (=? {:components {:entity-count {:measurement 3.0 :score 30}
+ :name-collisions {:measurement 2.0 :score 200}}}
+ (#'complexity/score-catalog es nil)))))
+
+ (testing "collision detection is case-insensitive and trims whitespace"
+ (let [es [(entity :name "Orders")
+ (entity :name " orders ")
+ (entity :name "ORDERS")]]
+ (is (=? {:components {:name-collisions {:measurement 2.0 :score 200}}}
+ (#'complexity/score-catalog es nil)))))
+
+ (testing "field count contributes +1 per field, summed across entities"
+ (let [es [(entity :name "a" :field-count 10)
+ (entity :name "b" :field-count 25)]]
+ (is (=? {:components {:field-count {:measurement 35.0 :score 35}}}
+ (#'complexity/score-catalog es nil)))))
+
+ (testing "repeated measures contribute +2 per repeat (measure name appearing on >1 entity)"
+ (let [es [(entity :name "invoices" :measure-names ["revenue" "discount"])
+ (entity :name "subscriptions" :measure-names ["revenue"])
+ (entity :name "products" :measure-names ["price"])]]
+ (is (=? {:components {:repeated-measures {:measurement 1.0 :score 2}}}
+ (#'complexity/score-catalog es nil)))))
+
+ (testing "nil embedder disables synonym scoring"
+ (let [es [(entity :name "customers") (entity :name "clients")]]
+ (is (=? {:components {:synonym-pairs {:measurement 0.0 :score 0}}}
+ (#'complexity/score-catalog es nil))))))
+
+(deftest ^:parallel synonym-scoring-test
+ (testing "cosine similarity above threshold flags a synonym pair"
+ (let [es [(entity :name "customers") (entity :name "clients")]
+ embedder (mock-embedder {"customers" [1.0 0.0 0.0]
+ "clients" [0.9 0.1 0.0]})]
+ (is (=? {:components {:synonym-pairs {:measurement 1.0 :score 50}}}
+ (#'complexity/score-catalog es embedder)))))
+
+ (testing "orthogonal embeddings produce no synonym pairs"
+ (let [es [(entity :name "customers") (entity :name "widgets")]
+ embedder (mock-embedder {"customers" [1.0 0.0]
+ "widgets" [0.0 1.0]})]
+ (is (=? {:components {:synonym-pairs {:measurement 0.0 :score 0}}}
+ (#'complexity/score-catalog es embedder)))))
+
+ (testing "exact-name duplicates don't double-count as synonym pairs"
+ (let [es [(entity :name "orders") (entity :name "orders") (entity :name "tickets")]
+ embedder (mock-embedder {"orders" [1.0 0.0]
+ "tickets" [0.0 1.0]})]
+ (is (=? {:components {:name-collisions {:measurement 1.0 :score 100}
+ :synonym-pairs {:measurement 0.0 :score 0}}}
+ (#'complexity/score-catalog es embedder)))))
+
+ (testing "entities without a vector from the embedder are simply skipped"
+ (let [es [(entity :name "customers") (entity :name "clients") (entity :name "ghost")]
+ ;; "ghost" is missing → not considered. The remaining two are synonyms.
+ embedder (mock-embedder {"customers" [1.0 0.0]
+ "clients" [0.99 0.01]})]
+ (is (=? {:components {:synonym-pairs {:measurement 1.0 :score 50}}}
+ (#'complexity/score-catalog es embedder)))))
+
+ (testing "embedder failure cascades nil through the catalog (no zero-fallback)"
+ (let [es [(entity :name "customers") (entity :name "clients")]
+ embedder (fn [_] (throw (ex-info "boom" {})))]
+ (is (=? {:total nil
+ :components {:synonym-pairs {:measurement nil :score nil :error "boom"}
+ ;; Sibling sub-scores still compute their real values — only the rollup
+ ;; cascades nil — so consumers can still see the unaffected dimensions.
+ :entity-count {:measurement 2.0 :score 20}}}
+ (#'complexity/score-catalog es embedder)))))
+
+ (testing "throwable with a nil/blank message still records :error as a nonblank string"
+ ;; Regression: we must keep :error present so an embedder failure is distinguishable from a
+ ;; genuine zero-synonym result. Fall back to the exception class name.
+ (let [es [(entity :name "customers") (entity :name "clients")]
+ synonym-of #(get-in (#'complexity/score-catalog es %) [:components :synonym-pairs])]
+ (doseq [[label embedder expected] [["nil message" (fn [_] (throw (NullPointerException.)))
+ "java.lang.NullPointerException"]
+ ["blank message" (fn [_] (throw (RuntimeException. " ")))
+ "java.lang.RuntimeException"]]]
+ (testing label
+ (let [sub (synonym-of embedder)]
+ (is (nil? (:score sub)))
+ (is (= expected (:error sub))
+ (format ":error must be a nonblank string when the throwable's message is %s" label))))))))
+
+(deftest ^:sequential complexity-scores-metabot-scope-opt-test
+ (testing ":verified-only? true flows the caller's metabot-scope through to enumerate-catalogs"
+ (let [captured-scope (atom nil)]
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (fn [scope]
+ (reset! captured-scope scope)
+ {:library []
+ :universe [(entity :name "orders") (entity :name "widgets")]
+ :metabot [(entity :name "orders")]})]
+ (let [{:keys [universe metabot]} (complexity/complexity-scores
+ :embedder nil
+ :metabot-scope {:verified-only? true :collection-id nil})]
+ (is (= {:verified-only? true :collection-id nil} @captured-scope)
+ "enumerate-catalogs was invoked with the caller's scope")
+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))
+ (is (= 2.0 (get-in universe [:components :entity-count :measurement])))))))
+ (testing ":collection-id alone also flows through to enumerate-catalogs (no verified flag required)"
+ (let [captured-scope (atom nil)]
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (fn [scope]
+ (reset! captured-scope scope)
+ {:library []
+ :universe [(entity :name "orders") (entity :name "widgets")]
+ :metabot [(entity :name "orders")]})]
+ (let [{:keys [metabot]} (complexity/complexity-scores
+ :embedder nil
+ :metabot-scope {:verified-only? false :collection-id 42})]
+ (is (= {:verified-only? false :collection-id 42} @captured-scope))
+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))))))
+ (testing "empty scope (or no :metabot-scope opt) still runs enumerate-catalogs with that scope so metabot's table-visibility filter still narrows the catalog"
+ ;; Regression: we used to reuse the :universe score verbatim when scope was empty. That hid the
+ ;; fact that Metabot's table visibility (`:visibility_type nil`, non-routed DB) already narrows
+ ;; the catalog even before Card scoping kicks in.
+ (doseq [scope [nil {} {:verified-only? false :collection-id nil}]]
+ (let [captured-scope (atom ::not-called)]
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (fn [s]
+ (reset! captured-scope s)
+ {:library []
+ :universe [(entity :name "orders") (entity :name "widgets")]
+ :metabot [(entity :name "orders")]})]
+ (let [{:keys [universe metabot]} (complexity/complexity-scores
+ :embedder nil
+ :metabot-scope scope)]
+ (is (= scope @captured-scope)
+ (format "enumerate-catalogs was invoked with the caller's (possibly empty) scope=%s"
+ (pr-str scope)))
+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))
+ (is (= 2.0 (get-in universe [:components :entity-count :measurement])))))))))
+
+(deftest ^:sequential metabot-catalog-excludes-hidden-tables-test
+ (testing ":metabot tables filter out hidden (`visibility_type` non-nil) and routed-DB tables so the
+ catalog matches what Metabot/search can actually surface"
+ ;; Regression: `:metabot` used to count every active non-audit table, which overcounted on
+ ;; instances with hidden tables or routed-database tables — Metabot never surfaces those.
+ (mt/with-temp
+ [:model/Database {router-db :id} {:name "Router DB"}
+ :model/Database {routed-db :id} {:name "Routed DB" :router_database_id router-db}
+ :model/Database {plain-db :id} {:name "Plain DB"}
+ :model/Table {visible :id} {:db_id plain-db :name "visible_table"
+ :active true :visibility_type nil}
+ :model/Table {hidden :id} {:db_id plain-db :name "hidden_table"
+ :active true :visibility_type "hidden"}
+ :model/Table {technical :id} {:db_id plain-db :name "technical_table"
+ :active true :visibility_type "technical"}
+ :model/Table {routed :id} {:db_id routed-db :name "routed_table"
+ :active true :visibility_type nil}]
+ (let [metabot-entities (:metabot (#'complexity/enumerate-catalogs nil))
+ ids (into #{} (comp (filter #(= :table (:kind %))) (map :id)) metabot-entities)]
+ (testing "visible non-routed table is included"
+ (is (contains? ids visible)))
+ (testing "hidden and technical tables are excluded"
+ (is (not (contains? ids hidden))
+ "tables with visibility_type=hidden are filtered out")
+ (is (not (contains? ids technical))
+ "tables with visibility_type=technical are filtered out"))
+ (testing "tables on a routed database are excluded"
+ (is (not (contains? ids routed))
+ "tables whose db has router_database_id are filtered out"))))))
+
+(deftest ^:sequential metabot-collection-scope-ids-test
+ (testing "nil collection-id returns nil (no Metabot collection scope configured)"
+ (is (nil? (#'complexity/metabot-collection-scope-ids nil))))
+ (testing "valid collection-id returns the id plus its descendants"
+ (mt/with-temp [:model/Collection {parent :id} {:name "Scope parent" :location "/"}
+ :model/Collection {child :id} {:name "Scope child" :location (format "/%d/" parent)}]
+ (is (= #{parent child}
+ (#'complexity/metabot-collection-scope-ids parent)))))
+ (testing "invalid/deleted collection-id still returns a singleton set with the raw id"
+ ;; The live `metabot-metrics-and-models-query` filters on the raw `collection_id` even when
+ ;; the collection row is missing (stale Metabot scope value). If we dropped the filter here,
+ ;; the :metabot catalog would overcount and drift back toward :universe.
+ (let [ghost-id Integer/MAX_VALUE]
+ (is (nil? (t2/select-one :model/Collection :id ghost-id))
+ "pre-check: the phantom id isn't actually a real collection")
+ (is (= #{ghost-id}
+ (#'complexity/metabot-collection-scope-ids ghost-id))))))
+
+(deftest ^:parallel fn-embedder-test
+ (testing "normalizes names, dedupes, zips vectors by position, and omits entries with no vector"
+ (let [known-vectors {"foo" (float-array [1.0 0.0])
+ "bar" (float-array [0.0 1.0])
+ "baz" (float-array [0.5 0.5])}
+ embedder (embedders/fn-embedder (partial mapv known-vectors))
+ result (embedder [{:name "Foo"}
+ {:name " BAR"}
+ {:name " Foo \t \n"}
+ {:name "missing"}])]
+ (is (=? {"foo" (known-vectors "foo")
+ "bar" (known-vectors "bar")}
+ result)))))
+
+(deftest ^:sequential library-empty-when-no-library-collection-test
+ (testing "on an instance with no Library collection, the library score is zero and universe still reports"
+ (collections.tu/without-library
+ (mt/with-temp [:model/Database {db-id :id} {:name "No-library Test DB"}
+ :model/Table _ {:db_id db-id :name "contributes_to_universe" :active true}]
+ (let [{:keys [library universe]} (complexity/complexity-scores :embedder nil)]
+ (testing "library is empty (no collection tree)"
+ (is (= {:total 0
+ :components {:entity-count {:measurement 0.0 :score 0}
+ :name-collisions {:measurement 0.0 :score 0}
+ :synonym-pairs {:measurement 0.0 :score 0}
+ :field-count {:measurement 0.0 :score 0}
+ :repeated-measures {:measurement 0.0 :score 0}}}
+ library)))
+ (testing "universe still enumerates appdb content (our temp table + whatever else is there)"
+ (is (pos? (:total universe)))))))))
+
+(deftest ^:sequential library-excludes-audit-content-test
+ (testing "published audit-db content in the Library tree is excluded so :library stays a subset of :universe"
+ ;; Regression: library-entities previously didn't filter on audit/audit-db-id while universe/metabot did.
+ ;; An audit card/table placed in the Library could push :library past :universe and break the new
+ ;; subset invariant (the hermetic tests below assume library ⊆ universe on every component).
+ (mt/with-temp [:model/Collection {lib-id :id} {:type collections/library-collection-type
+ :name "Library"
+ :location "/"}
+ :model/Collection {data-id :id} {:type collections/library-data-collection-type
+ :name "Data"
+ :location (format "/%d/" lib-id)}
+ :model/Collection {mets-id :id} {:type collections/library-metrics-collection-type
+ :name "Metrics"
+ :location (format "/%d/" lib-id)}
+ :model/Database {audit-db :id} {:name "Fake Audit DB"}
+ :model/Database {real-db :id} {:name "Non-audit DB"}
+ ;; Audit-db tables published into the Library tree — MUST be excluded.
+ :model/Table _ {:db_id audit-db :name "audit_events"
+ :active true :is_published true
+ :collection_id data-id}
+ ;; Audit-db metric card in the Library — MUST be excluded.
+ :model/Card _ {:database_id audit-db :type :metric :name "Audit Revenue"
+ :archived false :collection_id mets-id}
+ ;; Non-audit table in the Library — must still count.
+ :model/Table _ {:db_id real-db :name "orders"
+ :active true :is_published true
+ :collection_id data-id}
+ ;; Non-audit metric card in the Library — must still count.
+ :model/Card _ {:database_id real-db :type :metric :name "Real Revenue"
+ :archived false :collection_id mets-id}]
+ (with-redefs [audit/audit-db-id audit-db]
+ (let [{:keys [library]} (complexity/complexity-scores :embedder nil)]
+ (is (= 2.0 (get-in library [:components :entity-count :measurement]))
+ "only the two non-audit entities count (audit table + audit metric card excluded)"))))))
+
+;; We're only reading the method table via `methods`, not calling the impure `!` fn — safe in parallel.
+#_{:clj-kondo/ignore [:metabase/validate-deftest]}
+(deftest ^:parallel scoring-task-registered-test
+ (testing "loading the data-complexity-score init namespace registers the scoring task's `task/init!` method"
+ (is (contains? (methods task/init!)
+ :metabase-enterprise.data-complexity-score.task.complexity-score/DataComplexityScoring))))
+
+(deftest ^:parallel search-index-embedder-degrades-gracefully-test
+ (testing "returns {} when semantic-search index isn't available (no throw)"
+ (is (= {} (semantic-search/search-index-embedder
+ [(entity :name "orders" :kind :table)])))))
+
+(deftest ^:sequential complexity-scores-with-real-search-index-embedder-test
+ (testing "complexity-scores drives the real search-index-embedder + pgvector end-to-end"
+ ;; Self-gated on MB_PGVECTOR_DB_URL — CI without semantic-search infra skips this;
+ ;; locally with pgvector running it exercises the pgvector read path that all other
+ ;; embedder tests in this ns stub out (via mt/with-dynamic-fn-redefs on try-active-index-state /
+ ;; fetch-batch). Uses `:mock-indexed` so the *embedding model* is a lookup-table — but
+ ;; the resulting vectors are real rows in the index table that search-index-embedder
+ ;; queries via SQL.
+ (when semantic.db.datasource/db-url
+ (let [captured (atom nil)]
+ (mt/with-premium-features #{:semantic-search}
+ (mt/as-admin
+ (semantic.tu/with-test-db! {:mode :mock-indexed}
+ (reset! captured (complexity/complexity-scores
+ :embedder semantic-search/search-index-embedder)))))
+ (is (=? {:meta {:embedding-model {:provider "mock" :model-name "model"}}
+ :universe {:components {:synonym-pairs {:measurement number?
+ :score nat-int?}}}}
+ @captured)
+ "embedder returned vectors from pgvector and the synonym axis produced a real measurement")))))
+
+(deftest ^:sequential search-index-embedder-propagates-read-failures-test
+ (testing "pgvector read failures propagate so the caller can flag a degraded synonym axis"
+ ;; Regression guard: before this, the embedder swallowed read exceptions and returned {}, which
+ ;; looked indistinguishable from \"no synonym matches.\" A transient search-index failure
+ ;; silently underreported complexity with no `:error` on the Snowplow payload.
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock :table-name "t" :model nil})
+ ss.embedders/fetch-batch
+ (fn [& _] (throw (ex-info "pgvector read failed" {})))]
+ (testing "the embedder itself throws rather than swallowing the failure"
+ (is (thrown-with-msg? Throwable #"pgvector read failed"
+ (semantic-search/search-index-embedder
+ [{:id 1 :name "orders" :kind :table}]))))
+ (testing "score-synonym-pairs converts the propagated failure into :error on the sub-score"
+ (let [es [(entity :name "customers") (entity :name "clients")]]
+ (is (=? {:components {:synonym-pairs {:measurement nil
+ :score nil
+ :error string?}}}
+ (#'complexity/score-catalog es semantic-search/search-index-embedder))))))))
+
+(defn- stub-fetch-batch
+ "Build a `fetch-batch` stub backed by a map of expected pair-sets -> rows. Each
+ invocation looks up rows by the incoming `(model, model_id)` pairs, so the stub
+ validates the real batching contract (which pair-sets are requested) without
+ coupling to call order. Unknown or duplicate pair-sets throw. Returns
+ `{:unseen <atom>, :stub <fn>}`; after the call under test, assert
+ `(empty? @unseen)` to catch missing invocations."
+ [expected]
+ (let [unseen (atom (set (keys expected)))]
+ {:unseen unseen
+ :stub (fn [_ _ pairs]
+ (let [k (mapv vec pairs)]
+ (if (contains? @unseen k)
+ (do (swap! unseen disj k)
+ (get expected k))
+ (throw (ex-info (format "fetch-batch called with unexpected or duplicate pairs: %s (remaining expected: %s)"
+ (pr-str k) (pr-str @unseen))
+ {:pairs k :remaining @unseen})))))}))
+
+(deftest ^:sequential search-index-embedder-global-dedup-test
+ (testing "global dedup picks lowest numeric model_id regardless of batch boundaries"
+ ;; Stub at fetch-batch so the real batching path executes. With batch-size 1, each entity
+ ;; pair lands in its own SQL batch, so the two duplicates come back from separate partitions
+ ;; and must be merged by the fold. The stub keys expected batches by `(model, model_id)`
+ ;; pair-seq, so a regression in `entity-type->search-model` or any missing batch fails here.
+ (let [winner-vec (float-array [1.0 0.0 0.0])
+ loser-vec (float-array [0.0 1.0 0.0])
+ {:keys [unseen stub]}
+ (stub-fetch-batch
+ {;; table with model_id "10" (loser — higher id)
+ [["table" "10"]] [{:name "Orders" :model_id "10" :model "table" :embedding loser-vec}]
+ ;; table with model_id "2" (winner — lower id)
+ [["table" "2"]] [{:name "orders" :model_id "2" :model "table" :embedding winner-vec}]})]
+ (with-redefs [ss.embedders/fetch-batch-size 1]
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock :table-name "t" :model nil})
+ ss.embedders/fetch-batch stub]
+ (let [result (semantic-search/search-index-embedder
+ [{:id 10 :name "Orders" :kind :table}
+ {:id 2 :name "orders" :kind :table}])]
+ (is (= 1 (count result)) "duplicate normalized names collapse to one entry")
+ (is (= (seq winner-vec) (seq (get result "orders")))
+ "the row with the lowest numeric model_id wins")
+ (is (empty? @unseen)
+ "every expected fetch-batch pair-set must be requested"))))))
+
+ (testing "cross-model duplicates: lowest model_id wins, model is secondary tie-break"
+ ;; :kind :question maps to "card" and :kind :table maps to "table" via entity-type->search-model,
+ ;; so the real (model, model_id) query contract is exercised. Both have id 5, so model_ids tie
+ ;; and the lexicographically smaller model ("card" < "table") must win. The stub is keyed by
+ ;; pairs so a regression in entity-type->search-model (e.g. :question no longer mapping to
+ ;; "card") throws from the stub instead of passing silently.
+ (let [card-vec (float-array [1.0 0.0])
+ table-vec (float-array [0.0 1.0])
+ {:keys [unseen stub]}
+ (stub-fetch-batch
+ {;; card (from :question entity)
+ [["card" "5"]] [{:name "Revenue" :model_id "5" :model "card" :embedding card-vec}]
+ ;; table (from :table entity)
+ [["table" "5"]] [{:name "revenue" :model_id "5" :model "table" :embedding table-vec}]})]
+ (with-redefs [ss.embedders/fetch-batch-size 1]
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock :table-name "t" :model nil})
+ ss.embedders/fetch-batch stub]
+ (let [result (semantic-search/search-index-embedder
+ [{:id 5 :name "Revenue" :kind :question}
+ {:id 5 :name "revenue" :kind :table}])]
+ (is (= 1 (count result)))
+ (is (= (seq card-vec) (seq (get result "revenue")))
+ "when model_ids tie, the lexicographically smaller model wins (card < table)")
+ (is (empty? @unseen)
+ "every expected fetch-batch pair-set must be requested")))))))
+
+(deftest ^:sequential search-index-embedder-cross-batch-dedup-test
+ (testing "duplicates split across separate fetch batches are resolved globally"
+ ;; Mock at fetch-batch so the real partition-all fold executes. With batch-size 1, each entity
+ ;; pair lands in its own SQL batch. The stub is keyed by the expected `(model, model_id)`
+ ;; pair-seq, so it validates the batching contract — `entity-type->search-model` mapping
+ ;; (`:question` → "card", `:table` → "table") is exercised, not assumed.
+ (let [winner-vec (float-array [1.0 0.0 0.0])
+ loser-vec (float-array [0.0 1.0 0.0])
+ other-vec (float-array [0.0 0.0 1.0])
+ {:keys [unseen stub]}
+ (stub-fetch-batch
+ {;; table with model_id "7"
+ [["table" "7"]] [{:name "Orders" :model_id "7" :model "table" :embedding loser-vec}]
+ ;; card with model_id "3" — should win globally
+ [["card" "3"]] [{:name "orders" :model_id "3" :model "card" :embedding winner-vec}]
+ ;; different entity, no collision
+ [["table" "1"]] [{:name "Products" :model_id "1" :model "table" :embedding other-vec}]})]
+ (with-redefs [ss.embedders/fetch-batch-size 1]
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock :table-name "t" :model nil})
+ ss.embedders/fetch-batch stub]
+ (let [result (semantic-search/search-index-embedder
+ [{:id 7 :name "Orders" :kind :table}
+ {:id 3 :name "orders" :kind :question}
+ {:id 1 :name "Products" :kind :table}])]
+ (is (= 2 (count result)) "two distinct normalized names survive dedup")
+ (is (= (seq winner-vec) (seq (get result "orders")))
+ "lowest model_id wins across batch boundaries")
+ (is (some? (get result "products"))
+ "non-colliding entity is retained")
+ (is (empty? @unseen)
+ "every expected fetch-batch pair-set must be requested"))))))
+
+ (testing "cross-batch, cross-model duplicates: model_id primary, model secondary"
+ ;; Same normalized name from three different batches and three different model types.
+ ;; model_id "5" appears twice (card + dataset); model_id "12" is in a third batch.
+ ;; Expected winner: model_id "5", model "card" (lowest id, then lexicographic: "card" < "dataset").
+ ;; Entity kinds cover :table → "table", :model → "dataset", :question → "card".
+ (let [winner-vec (float-array [1.0 0.0])
+ same-id-other-model (float-array [0.0 1.0])
+ higher-id-vec (float-array [0.5 0.5])
+ {:keys [unseen stub]}
+ (stub-fetch-batch
+ {;; high model_id, model "table"
+ [["table" "12"]] [{:name "Revenue" :model_id "12" :model "table" :embedding higher-id-vec}]
+ ;; low model_id, model "dataset"
+ [["dataset" "5"]] [{:name "revenue" :model_id "5" :model "dataset" :embedding same-id-other-model}]
+ ;; low model_id, model "card" — card < dataset so this wins
+ [["card" "5"]] [{:name "REVENUE" :model_id "5" :model "card" :embedding winner-vec}]})]
+ (with-redefs [ss.embedders/fetch-batch-size 1]
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock :table-name "t" :model nil})
+ ss.embedders/fetch-batch stub]
+ (let [result (semantic-search/search-index-embedder
+ [{:id 12 :name "Revenue" :kind :table}
+ {:id 5 :name "revenue" :kind :model}
+ {:id 5 :name "REVENUE" :kind :question}])]
+ (is (= 1 (count result)) "all three collapse to one normalized name")
+ (is (= (seq winner-vec) (seq (get result "revenue")))
+ "model_id 5 + model card wins (lowest id, then lexicographic: card < dataset)")
+ (is (empty? @unseen)
+ "every expected fetch-batch pair-set must be requested")))))))
+
+(deftest ^:parallel prefer-new-row-test
+ (let [prefer? #'ss.embedders/prefer-new-row?]
+ (are [expected? new-row prior-row]
+ (expected? (prefer? new-row prior-row))
+ ;; lower numeric model_id wins
+ true? {:mid 2 :model_id "2" :model "card"} {:mid 10 :model_id "10" :model "card"}
+ false? {:mid 10 :model_id "10" :model "card"} {:mid 2 :model_id "2" :model "card"}
+ ;; equal model_ids tie-break on model name
+ true? {:mid 5 :model_id "5" :model "card"} {:mid 5 :model_id "5" :model "table"}
+ false? {:mid 5 :model_id "5" :model "table"} {:mid 5 :model_id "5" :model "card"}
+ ;; same parsed number but different raw strings → tie-break on raw model_id before model
+ true? {:mid 2 :model_id "02" :model "card"} {:mid 2 :model_id "2" :model "card"}
+ false? {:mid 2 :model_id "2" :model "card"} {:mid 2 :model_id "02" :model "card"}
+ ;; numeric always beats non-numeric
+ true? {:mid 99 :model_id "99" :model "card"} {:mid nil :model_id "abc" :model "card"}
+ false? {:mid nil :model_id "abc" :model "card"} {:mid 1 :model_id "1" :model "card"}
+ ;; both non-numeric: lexicographic on model_id string
+ true? {:mid nil :model_id "abc" :model "card"} {:mid nil :model_id "xyz" :model "card"}
+ false? {:mid nil :model_id "xyz" :model "card"} {:mid nil :model_id "abc" :model "card"})))
+
+(deftest ^:sequential meta-embedding-model-absent-when-unavailable-test
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [] :universe [] :metabot []})]
+ (testing ":embedding-model key is absent from :meta when the search index is unreachable"
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state (constantly nil)]
+ ;; Pass the real search-index-embedder so the identity check in complexity-scores succeeds,
+ ;; but the embedder returns {} because try-active-index-state is nil.
+ (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]
+ (is (not (contains? meta :embedding-model))))))
+ (testing ":embedding-model key is present in :meta when the active model is non-nil"
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock
+ :table-name "mock_table"
+ :model {:provider "openai"
+ :model-name "text-embedding-3-small"}})
+ ss.embedders/fetch-batch (constantly [])]
+ (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]
+ (is (= {:provider "openai" :model-name "text-embedding-3-small"}
+ (:embedding-model meta))))))))
+
+(deftest ^:sequential active-embedding-model-reads-from-active-index-test
+ (testing "active-embedding-model returns the model from the active index, not the configured setting"
+ (let [active-model {:provider "openai" :model-name "text-embedding-ada-002"}]
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock
+ :table-name "mock_table"
+ :model active-model})]
+ (is (= {:provider "openai" :model-name "text-embedding-ada-002"}
+ (semantic-search/active-embedding-model))))))
+ (testing "active-embedding-model returns nil when the index state has no model"
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock
+ :table-name "mock_table"
+ :model nil})]
+ (is (nil? (semantic-search/active-embedding-model)))))
+ (testing "active-embedding-model returns nil when the index is unreachable"
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state (constantly nil)]
+ (is (nil? (semantic-search/active-embedding-model))))))
+
+(defn- complexity-events!
+ "Drain the fake Snowplow collector and return only complexity events."
+ []
+ (->> (snowplow-test/pop-event-data-and-user-id!)
+ (map :data)
+ (filter #(= "data_complexity_scoring" (get % "event")))))
+
+(def ^:private leaf-key->group
+ "Mirrors `complexity/component->group` for test assertions. Kept in snake-case string form
+ (matching how keys land in the Snowplow payload)."
+ {"entity_count" "size"
+ "field_count" "size"
+ "name_collisions" "ambiguity"
+ "synonym_pairs" "ambiguity"
+ "repeated_measures" "ambiguity"})
+
+(defn- snake [k] (-> k name (str/replace "-" "_")))
+
+(defn- expected-keys-for-catalog
+ "For a catalog result, return `#{[key score], ...}` matching what emit-snowplow! should emit:
+ grand `total`, one `<group>.total` per group, and one `<group>.<leaf>` per sub-component."
+ [{:keys [total components]}]
+ (let [leaves (into {} (map (fn [[k sub]] [(snake k) (:score sub)]) components))
+ group-total (reduce-kv (fn [acc leaf score]
+ (update acc (leaf-key->group leaf) (fnil + 0) score))
+ {} leaves)]
+ (set (concat [["total" total]]
+ (for [[g s] group-total] [(str g ".total") s])
+ (for [[leaf s] leaves] [(str (leaf-key->group leaf) "." leaf) s])))))
+
+(deftest ^:sequential emit-snowplow-publishes-total-and-each-subscore-test
+ (testing "one event per (catalog × key) — grand total, group rollups, and leaves — with correct scores"
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")
+ (entity :name "customers")]
+ :universe [(entity :name "orders")
+ (entity :name "customers")
+ (entity :name "widgets")]
+ :metabot []})]
+ ;; Drain any startup/setting events so we only assert on emissions from the call below.
+ (snowplow-test/pop-event-data-and-user-id!)
+ (let [{:keys [library universe metabot]} (complexity/complexity-scores :embedder nil)
+ events (complexity-events!)
+ expected (into #{}
+ (for [[catalog result] {"library" library
+ "universe" universe
+ "metabot" metabot}
+ [k score] (expected-keys-for-catalog result)]
+ [catalog k score]))
+ actual (into #{}
+ (map (fn [e] [(get e "catalog") (get e "key") (get e "score")]) events))]
+ (is (= (count expected) (count events))
+ "every (catalog, key) is emitted exactly once — no duplicates")
+ (is (= expected actual)
+ "every (catalog, key) pair carries the matching score from the result")
+ (testing "every event carries the event name, formula version, and parameters map"
+ (is (every? (fn [e]
+ (and (= "data_complexity_scoring" (get e "event"))
+ (integer? (get e "formula_version"))
+ (map? (get e "parameters"))
+ (number? (get-in e ["parameters" "synonym_threshold"]))))
+ events))))))))
+
+(deftest ^:sequential emit-snowplow-includes-measurement-for-sub-components-test
+ (testing "each leaf event carries the raw pre-score measurement; totals and group-totals do not"
+ (snowplow-test/with-fake-snowplow-collector
+ ;; Library: 3 entities, one collision pair, 5 fields total.
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders" :field-count 2)
+ (entity :name "orders" :field-count 0)
+ (entity :name "widgets" :field-count 3)]
+ :universe []
+ :metabot []})]
+ (snowplow-test/pop-event-data-and-user-id!)
+ (complexity/complexity-scores :embedder nil)
+ (let [by-key (->> (complexity-events!)
+ (filter #(= "library" (get % "catalog")))
+ (into {} (map (juxt #(get % "key") identity))))]
+ (testing "aggregate totals (grand and per-group) have no measurement key"
+ (is (not (contains? (get by-key "total") "measurement")))
+ (is (not (contains? (get by-key "size.total") "measurement")))
+ (is (not (contains? (get by-key "ambiguity.total") "measurement"))))
+ (testing "size leaves carry raw counts"
+ (is (= 3.0 (get-in by-key ["size.entity_count" "measurement"])))
+ (is (= 5.0 (get-in by-key ["size.field_count" "measurement"]))))
+ (testing "ambiguity leaves carry their respective measurements"
+ (is (= 1.0 (get-in by-key ["ambiguity.name_collisions" "measurement"])))
+ (is (= 0.0 (get-in by-key ["ambiguity.synonym_pairs" "measurement"])))
+ (is (= 0.0 (get-in by-key ["ambiguity.repeated_measures" "measurement"])))))))))
+
+(deftest ^:sequential emit-snowplow-cascades-nil-on-embedder-failure-test
+ (testing "embedder failure cascades nil through aggregates without skipping any events"
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "customers")
+ (entity :name "clients")]
+ :universe []
+ :metabot []})]
+ (snowplow-test/pop-event-data-and-user-id!)
+ (complexity/complexity-scores :embedder (fn [_] (throw (ex-info "embedder boom" {}))))
+ (let [by-key (->> (complexity-events!)
+ (filter #(= "library" (get % "catalog")))
+ (into {} (map (juxt #(get % "key") identity))))]
+ (testing ":error from the synonym-pair scorer reaches the Snowplow leaf event"
+ (is (= "embedder boom" (get-in by-key ["ambiguity.synonym_pairs" "error"]))))
+ (testing ":error is only present on the originating leaf — aggregates use null score instead"
+ (is (not-any? #(contains? % "error")
+ (vals (dissoc by-key "ambiguity.synonym_pairs")))))
+ (testing "the failed leaf publishes null :score (no zero-fallback)"
+ (is (nil? (get-in by-key ["ambiguity.synonym_pairs" "score"]))))
+ (testing "aggregates that include the failed leaf cascade null :score"
+ (is (nil? (get-in by-key ["ambiguity.total" "score"])))
+ (is (nil? (get-in by-key ["total" "score"]))))
+ (testing "unaffected aggregates keep their numeric :score (cascade is leaf-scoped, not catalog-wide)"
+ ;; size group has no synonym dependency, so its rollup must still be a real number even
+ ;; when ambiguity falls through. Catches a regression where the cascade goes too far.
+ (is (number? (get-in by-key ["size.total" "score"])))))))))
+
+(deftest ^:sequential emit-snowplow-truncates-error-to-schema-max-test
+ (testing "a pathologically long exception message is truncated so it doesn't fail schema validation"
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "customers")
+ (entity :name "clients")]
+ :universe []
+ :metabot []})]
+ (snowplow-test/pop-event-data-and-user-id!)
+ (let [huge (apply str (repeat 5000 "x"))]
+ (complexity/complexity-scores :embedder (fn [_] (throw (ex-info huge {}))))
+ (let [err (->> (complexity-events!)
+ (filter #(= "ambiguity.synonym_pairs" (get % "key")))
+ first
+ (#(get % "error")))]
+ (is (= 1024 (count err))
+ "error is clipped to the schema's maxLength of 1024")))))))
+
+(deftest ^:sequential emit-snowplow-includes-embedding-model-meta-test
+ (testing "every event's parameters carry embedding_model_provider/name when the search-index embedder is active"
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
+ (constantly {:pgvector :mock
+ :table-name "mock_table"
+ :model {:provider "openai"
+ :model-name "text-embedding-3-small"}})
+ ss.embedders/fetch-batch (constantly [])
+ complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot []})]
+ (snowplow-test/pop-event-data-and-user-id!)
+ (complexity/complexity-scores :embedder semantic-search/search-index-embedder)
+ (let [events (complexity-events!)]
+ (is (seq events) "sanity: events were emitted")
+ (is (every? #(= "openai" (get-in % ["parameters" "embedding_model_provider"])) events))
+ (is (every? #(= "text-embedding-3-small" (get-in % ["parameters" "embedding_model_name"])) events)))))))
+
+(deftest ^:sequential emit-snowplow-failure-is-swallowed-test
+ (testing "emission failure is caught; complexity-scores still returns the score and logs a warning"
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot []})
+ analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :warn]]
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (=? {:library {:total 10 :components {:entity-count {:measurement 1.0 :score 10}}}
+ :universe {:total 10 :components {:entity-count {:measurement 1.0 :score 10}}}}
+ result))
+ (is (some #(re-find #"Failed to publish complexity score" (:message %))
+ (messages))
+ "a warning about the publish failure was logged"))))))
+
+(deftest ^:sequential local-info-log-is-emitted-even-when-snowplow-fails-test
+ (testing "the 'Semantic complexity score' info log fires independently of Snowplow emission"
+ ;; Guards two regressions together: local logging being removed, and local logging being
+ ;; gated on successful telemetry (so a broken collector would silence the operator-visible log).
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot []})
+ analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]
+ (complexity/complexity-scores :embedder nil)
+ (is (some #(and (= :info (:level %))
+ (re-find #"Semantic complexity score" (:message %)))
+ (messages))
+ "the score was logged locally at :info even though Snowplow emission threw")))))
+
+(deftest ^:sequential scheduled-task-logs-score-test
+ (testing "the scheduled task body runs complexity-scores so operators see a score line in the logs"
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [(entity :name "orders")]
+ :universe [(entity :name "orders")]
+ :metabot []})]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true]
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]
+ (#'task.complexity-score/run-scoring! "test-fp")
+ (is (some #(and (= :info (:level %))
+ (re-find #"Semantic complexity score" (:message %)))
+ (messages))
+ "the scheduled task produced the expected info log"))))))
+
+(deftest ^:sequential complexity-score-library-hermetic-test
+ (testing "library score is computed over exactly the Library collection tree — known inputs produce known scores"
+ ;; The library tree gets a fixed set of tables, fields, measures, and metric cards. One extra
+ ;; collection + table + card sit outside the library so the universe is a strict superset.
+ (mt/with-temp
+ [:model/Collection {lib-id :id} {:type collections/library-collection-type
+ :name "Library"
+ :location "/"}
+ :model/Collection {data-id :id} {:type collections/library-data-collection-type
+ :name "Data"
+ :location (format "/%d/" lib-id)}
+ :model/Collection {metrics-id :id} {:type collections/library-metrics-collection-type
+ :name "Metrics"
+ :location (format "/%d/" lib-id)}
+ :model/Collection {other-id :id} {:name "Outside" :location "/"}
+ :model/Database {db-id :id} {:name "Hermetic Library DB"}
+ ;; Library tables. clients + customers are a synonym pair for the mock embedder below.
+ :model/Table {t1 :id} {:db_id db-id :name "orders"
+ :active true :is_published true :collection_id data-id}
+ :model/Table {t2 :id} {:db_id db-id :name "subscriptions"
+ :active true :is_published true :collection_id data-id}
+ :model/Table _ {:db_id db-id :name "clients"
+ :active true :is_published true :collection_id data-id}
+ :model/Table _ {:db_id db-id :name "customers"
+ :active true :is_published true :collection_id data-id}
+ ;; Non-library tables: active (→ in universe) but not published into the library tree.
+ ;; audit_events + audit_log are another synonym pair via the mock embedder.
+ :model/Table {t-audit :id} {:db_id db-id :name "audit_events"
+ :active true :is_published false :collection_id other-id}
+ :model/Table _ {:db_id db-id :name "audit_log"
+ :active true :is_published false :collection_id other-id}
+ ;; Library fields: 2 on orders, 1 on subscriptions.
+ :model/Field _ {:table_id t1 :name "id" :active true :base_type :type/Integer}
+ :model/Field _ {:table_id t1 :name "total" :active true :base_type :type/Float}
+ :model/Field _ {:table_id t2 :name "id" :active true :base_type :type/Integer}
+ ;; Non-library fields on audit_events → bumps universe field-count above library.
+ :model/Field _ {:table_id t-audit :name "audit_id"
+ :active true :base_type :type/Integer}
+ :model/Field _ {:table_id t-audit :name "audit_ts"
+ :active true :base_type :type/DateTime}
+ ;; Library measures: "revenue" on two library tables → one repeated-measure pair in library.
+ :model/Measure _ {:table_id t1 :name "revenue"
+ :creator_id (mt/user->id :rasta)
+ :definition {} :archived false}
+ :model/Measure _ {:table_id t2 :name "revenue"
+ :creator_id (mt/user->id :rasta)
+ :definition {} :archived false}
+ ;; Non-library measure with the same name → bumps universe repeated-measures above library.
+ :model/Measure _ {:table_id t-audit :name "revenue"
+ :creator_id (mt/user->id :rasta)
+ :definition {} :archived false}
+ ;; Two library metric cards named "Revenue" → one name-collision pair in library.
+ :model/Card _ {:database_id db-id :type :metric :name "Revenue"
+ :archived false :collection_id metrics-id}
+ :model/Card _ {:database_id db-id :type :metric :name "Revenue"
+ :archived false :collection_id metrics-id}
+ ;; Non-library card also named "Revenue" → 3 "Revenue" entities in universe → 2 collision pairs.
+ :model/Card _ {:database_id db-id :type :model :name "Revenue"
+ :archived false :collection_id other-id}]
+ ;; Each name gets its own orthogonal dimension so only the intended pairs cluster.
+ (let [embedder (mock-embedder {"orders" [1.0 0.0 0.0 0.0 0.0 0.0 0.0]
+ "subscriptions" [0.0 1.0 0.0 0.0 0.0 0.0 0.0]
+ "clients" [0.0 0.0 1.0 0.0 0.0 0.0 0.0]
+ "customers" [0.0 0.0 0.99 0.1 0.0 0.0 0.0] ; ≈ clients (library)
+ "revenue" [0.0 0.0 0.0 0.0 1.0 0.0 0.0] ; "Revenue" cards normalize here
+ "audit_events" [0.0 0.0 0.0 0.0 0.0 1.0 0.0]
+ "audit_log" [0.0 0.0 0.0 0.0 0.0 0.99 0.1]}) ; ≈ audit_events (universe-only)
+ {:keys [library universe]} (complexity/complexity-scores :embedder embedder)]
+ (testing "library reflects exactly what we put in the Library collection tree"
+ ;; Library: 4 tables + 2 metric cards = 6 entities.
+ ;; entity-count 6 × 10 = 60
+ ;; name-collisions "revenue" (2 metric cards) = 1 pair × 100 = 100
+ ;; synonym-pairs clients ↔ customers = 1 × 50 = 50
+ ;; field-count orders(2) + subscriptions(1) + others(0) = 3 × 1 = 3
+ ;; repeated-measures "revenue" on orders + subscriptions = 1 × 2 = 2
+ ;; total 60 + 100 + 50 + 3 + 2 = 215
+ (is (= {:total 215
+ :components {:entity-count {:measurement 6.0 :score 60}
+ :name-collisions {:measurement 1.0 :score 100}
+ :synonym-pairs {:measurement 1.0 :score 50}
+ :field-count {:measurement 3.0 :score 3}
+ :repeated-measures {:measurement 1.0 :score 2}}}
+ library)))
+ (testing "universe is a strict superset of library on every component: every measurement and score is higher"
+ ;; Note: :synonym-pairs is monotonic on this fixture but not in general — score-synonym-pairs
+ ;; dedupes by normalized name and picks one embedding per name, so a universe-only entity
+ ;; sharing a normalized name with a library entity could in theory flip which vector wins and
+ ;; decrease the pair count/score. Our fixture doesn't hit that case; if this assertion ever
+ ;; flakes, that's the reason.
+ (doseq [component [:entity-count :name-collisions :synonym-pairs :field-count :repeated-measures]
+ k [:measurement :score]
+ :let [lib-v (get-in library [:components component k])
+ uni-v (get-in universe [:components component k])]]
+ (is (> uni-v lib-v)
+ (format "universe %s %s (%s) should be strictly > library %s %s (%s)"
+ component k uni-v component k lib-v))))
+ (testing "universe total is strictly higher than library total"
+ (is (> (:total universe) (:total library))))))))
+
+(defn- stub-result
+ "Build a `complexity/complexity-scores` stand-in whose metadata records whether publishing worked."
+ [published?]
+ (with-meta
+ {:library {:total 0 :components {}} :universe {:total 0 :components {}}
+ :metabot {:total 0 :components {}} :meta {}}
+ {:metabase-enterprise.data-complexity-score.complexity/snowplow-published? published?}))
+
+(deftest ^:sequential run-scoring-persists-fingerprint-only-on-successful-publish-test
+ (testing "fingerprint advances only when Snowplow accepted the event — failed publish must leave
+ the stale fingerprint in place so the next boot / cron retries"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ (testing "successful publish → fingerprint advances to the claim's fingerprint"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]
+ (#'task.complexity-score/run-scoring! "fresh-fp")
+ (is (= "fresh-fp" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint stamped from the claim fingerprint, not re-sampled at commit time"))))
+ (testing "failed publish → fingerprint stays at the stale value for the next retry"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]
+ (#'task.complexity-score/run-scoring! "fresh-fp")
+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint preserved — next boot / cron will retry the emission")))))))
+
+(deftest ^:sequential maybe-emit-boot-score-only-advances-fingerprint-on-successful-publish-test
+ (testing "boot-time emission never advances the last-successful fingerprint on failure — the
+ success fingerprint is separate from the in-progress scoring claim, so a scoring/publish
+ failure (or a crash mid-run) leaves the fingerprint stale and the next boot retries"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ (testing "publish failure → fingerprint stays at the stale value (and claim is cleared)"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint unchanged — next boot/cron will retry the emission")
+ (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))
+ "scoring claim released so other paths can proceed without waiting for TTL"))))
+ (testing "publish success → fingerprint advances to the new value (and claim is cleared)"
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint advanced to reflect the confirmed publish")
+ (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))
+ "scoring claim released after successful run")))))))
+
+(deftest ^:sequential maybe-emit-boot-score-skips-when-another-path-holds-active-claim-test
+ (testing "a fresh scoring claim (from a sibling node OR a concurrent cron tick) blocks duplicate
+ emission on this node, even when the last-successful fingerprint is stale — prevents
+ the boot and cron paths from both computing and publishing for the same fingerprint"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so
+ ;; it's well inside the TTL.
+ active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim active-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (false? @scoring-ran?)
+ "scoring skipped because another path already claimed the current fingerprint")
+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint untouched when the claim is skipped")
+ (is (= active-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ "other path's claim is preserved (we never took it, so we don't clear it)")))))))
+
+(deftest ^:sequential maybe-emit-boot-score-reclaims-when-prior-claim-has-expired-test
+ (testing "an expired (TTL-exceeded) scoring claim must not permanently suppress scoring — a
+ crashed or orphaned claim from a prior run should be replaced and scoring proceed"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.
+ expired-claim (pr-str {:fingerprint current-fp
+ :claimed-at (- (#'task.complexity-score/now-ms)
+ (* 60 60 1000))})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim expired-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (true? @scoring-ran?)
+ "scoring ran because the prior claim had aged past the TTL")
+ (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ "fingerprint advanced on successful publish after re-claim")))))))
+
+(deftest ^:sequential maybe-emit-boot-score-does-not-clear-sibling-claim-after-ttl-takeover-test
+ (testing "if our run outlives the TTL and another path legitimately re-claims, the original
+ claimant's release-scoring-claim! must NOT wipe the replacement claim — doing so would
+ reopen the duplicate-publish race (a third node/path would see no claim and run again)"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a
+ ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min
+ ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.
+ (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)
+ :claimed-at (System/currentTimeMillis)
+ :owner "sibling-node-owner-token"})]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim ""]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _]
+ (data-complexity-score.settings/data-complexity-scoring-claim! sibling-claim)
+ (stub-result true))]
+ (task.complexity-score/maybe-emit-boot-score!)
+ (is (= sibling-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ "replacement claim preserved — our release was a compare-and-clear and the owners didn't match")))))))
+
+(deftest ^:sequential cron-skips-when-boot-run-holds-active-claim-test
+ (testing "the cron job's scoring call skips when the boot-time run is already mid-flight for the
+ current fingerprint — the shared claim protocol serializes the two paths so they can't
+ both compute and publish in parallel"
+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]
+ (let [current-fp (#'task.complexity-score/current-fingerprint)
+ boot-claim (pr-str {:fingerprint current-fp
+ :claimed-at (#'task.complexity-score/now-ms)
+ :owner "boot-path-owner-token"})
+ scoring-ran? (atom false)]
+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true
+ data-complexity-scoring-last-fingerprint "stale"
+ data-complexity-scoring-claim boot-claim]
+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores
+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]
+ ;; Exercise the cron's code path via the shared helper — this is exactly what the
+ ;; Quartz job body calls. Re-sampling current-fingerprint does not matter because the
+ ;; live value matches the one the boot path claimed.
+ (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-scoring!)
+ (is (false? @scoring-ran?)
+ "cron tick skipped because the boot run holds the scoring claim")
+ (is (= boot-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ "boot's claim preserved — cron never took it, so it doesn't clear it")))))))
+
+(deftest ^:sequential complexity-scores-tags-publish-success-on-result-test
+ (testing "complexity-scores stamps publish success/failure via metadata for schedule/boot callers"
+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
+ (constantly {:library [] :universe [] :metabot []})]
+ (testing "successful publish → ::snowplow-published? true"
+ (snowplow-test/with-fake-snowplow-collector
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result)))))))
+ (testing "publish throw → ::snowplow-published? false (and the result is still returned)"
+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result)))))))
+ (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"
+ ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would
+ ;; look indistinguishable from a successful publish and the fingerprint would advance,
+ ;; silently suppressing the boot-time retry once tracking is turned back on.
+ (mt/with-temporary-setting-values [anon-tracking-enabled false]
+ (let [result (complexity/complexity-scores :embedder nil)]
+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?
+ (meta result))))))))))
diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0 b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0
new file mode 100644
index 00000000000..753f49dade4
--- /dev/null
+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0
@@ -0,0 +1,81 @@
+{
+ "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+ "description": "Data Complexity Score — one event per (catalog, key) slice.",
+ "self": {
+ "vendor": "com.metabase",
+ "name": "data_complexity",
+ "format": "jsonschema",
+ "version": "1-0-0"
+ },
+ "type": "object",
+ "required": [
+ "event",
+ "catalog",
+ "key",
+ "score",
+ "formula_version",
+ "parameters"
+ ],
+ "properties": {
+ "event": {
+ "description": "Event name.",
+ "type": "string",
+ "enum": [
+ "data_complexity_scoring"
+ ],
+ "maxLength": 64
+ },
+ "catalog": {
+ "description": "Which catalog of the data layer was scored.",
+ "type": "string",
+ "enum": [
+ "library",
+ "universe",
+ "metabot"
+ ],
+ "maxLength": 16
+ },
+ "key": {
+ "description": "Dotted identifier for this slice: `total`, a `<group>.total` rollup, or a `<group>.<component>` leaf. Free-form so new keys can be added without a schema bump.",
+ "type": "string",
+ "maxLength": 64
+ },
+ "score": {
+ "description": "Weighted score for this key. Null when the sub-score (or any sub-score it aggregates) couldn't be computed.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "measurement": {
+ "description": "Raw pre-score measurement (absent on aggregates; null on uncomputed leaves). Number, not integer, to future-proof.",
+ "type": [
+ "number",
+ "null"
+ ],
+ "minimum": 0,
+ "maximum": 9007199254740991
+ },
+ "error": {
+ "description": "Failure message on the originating leaf when its sub-score couldn't be computed; aggregates carry null `score` instead.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "maxLength": 1024
+ },
+ "formula_version": {
+ "description": "Scoring formula version; bumps on breaking changes. Kept top-level so cross-version filters are a single column.",
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 2147483647
+ },
+ "parameters": {
+ "description": "Scoring inputs that are likely to evolve (tunables, embedder identity). Sorted-key JSON so stable for SQL group-by; Metabase Transforms can break out its fields without bumping schema.",
+ "type": "object",
+ "additionalProperties": true
+ }
+ }
+}
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 5165745d1f3..dcd4dcef4b3 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -400,6 +400,16 @@
(prometheus/counter :metabase-search/semantic-indexer-dlq-failures
{:description "Number of failed semantic search DLQ retries"})
+ ;; data-complexity-score timing
+ ;; 1ms → 1min buckets; widen later if real-world runs push past a minute.
+ (prometheus/histogram :metabase-data-complexity/scoring-duration-ms
+ {:description "Duration (ms) of a full Data Complexity Score run end-to-end (enumerate + score for all three catalogs + Snowplow publish)."
+ :buckets [1 10 50 100 500 1000 5000 10000 30000 60000]})
+ (prometheus/histogram :metabase-data-complexity/phase-duration-ms
+ {:description "Duration (ms) of one stage (`enumerate` = DB fetch, `score` = in-memory scoring) for one catalog within a Data Complexity Score run."
+ :labels [:stage :catalog]
+ :buckets [1 10 50 100 500 1000 5000 10000 30000 60000]})
+
;; notification metrics
(prometheus/counter :metabase-notification/send-ok
{:description "Number of successful notification sends."
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.clj
index 1906b99d1d2..d4152c7c881 100644
--- a/src/metabase/analytics/snowplow.clj
+++ b/src/metabase/analytics/snowplow.clj
@@ -54,7 +54,8 @@
:snowplow/serialization "1-0-1"
:snowplow/simple_event "1-0-0"
:snowplow/cleanup "1-0-0"
- :snowplow/ai_service_event "1-0-0"})
+ :snowplow/ai_service_event "1-0-0"
+ :snowplow/data_complexity "1-0-0"})
(def ^:private SnowplowSchema
"Malli enum for valid Snowplow schemas"
@@ -144,20 +145,24 @@
[tracker event]
(.track ^Tracker tracker ^SelfDescribing event))
-(mu/defn track-event!
+(mu/defn track-event! :- :boolean
"Send a single analytics event to the Snowplow collector, if tracking is enabled for this MB instance and a collector
- is available."
+ is available. Returns true when the event was actually handed to the tracker; false when tracking is disabled or
+ emission threw — callers that need to gate durable side-effects on real delivery can check the return value."
([schema :- SnowplowSchema data]
(track-event! schema data api/*current-user-id*))
([schema :- SnowplowSchema data user-id]
- (when (analytics.settings/snowplow-enabled)
- (try
- (let [^SelfDescribing$Builder2 builder (-> (. SelfDescribing builder)
- (.eventData (payload schema (schema->version schema) data))
- (.customContext [(context)])
- (cond-> user-id (.subject (subject user-id))))
- ^SelfDescribing event (.build builder)]
- (track-event-impl! tracker event))
- (catch Throwable e
- (log/errorf e "Error sending Snowplow analytics event for schema %s" schema))))))
+ (boolean
+ (when (analytics.settings/snowplow-enabled)
+ (try
+ (let [^SelfDescribing$Builder2 builder (-> (. SelfDescribing builder)
+ (.eventData (payload schema (schema->version schema) data))
+ (.customContext [(context)])
+ (cond-> user-id (.subject (subject user-id))))
+ ^SelfDescribing event (.build builder)]
+ (track-event-impl! tracker event)
+ true)
+ (catch Throwable e
+ (log/errorf e "Error sending Snowplow analytics event for schema %s" schema)
+ false))))))
diff --git a/src/metabase/collections/core.clj b/src/metabase/collections/core.clj
index c16a5e34850..16c916b38cd 100644
--- a/src/metabase/collections/core.clj
+++ b/src/metabase/collections/core.clj
@@ -17,7 +17,9 @@
create-library-collection!
descendant-ids
library-collection
+ library-collection-type
library-data-collection-type
+ library-metrics-collection-type
location-path
moving-from-remote-synced?
moving-into-remote-synced?