The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Data Complexity Score (#72681)
Chris Truterd51c5d501290
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f0b8dc31e9d..30ada1b6b54 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1192,6 +1192,7 @@{:team "Metabot":api #{metabase.metabot.apimetabase.metabot.api.entity-analysis+ metabase.metabot.configmetabase.metabot.coremetabase.metabot.initmetabase.metabot.self@@ -1237,7 +1238,7 @@transforms-baseutilwarehouses}- :model-exports #{:model/AiUsageLog :model/MetabotMessage}+ :model-exports #{:model/AiUsageLog :model/Metabot :model/MetabotMessage}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -2814,6 +2815,31 @@:api #{}:uses #{premium-features}}+ enterprise/data-complexity-score+ {:team "Metabot"+ :api #{metabase-enterprise.data-complexity-score.api+ metabase-enterprise.data-complexity-score.init}+ :uses #{analytics+ api+ app-db+ audit-app+ collections+ config+ metabot+ premium-features+ settings+ startup+ task+ util+ enterprise/semantic-search}+ :model-imports #{:model/Card+ :model/Collection+ :model/Database+ :model/Measure+ :model/Metabot+ :model/ModerationReview+ :model/Table}}+enterprise/data-studio{:team "UX West":api #{metabase-enterprise.data-studio.api}@@ -3177,6 +3203,7 @@configconnection-poolllm+ metabotmodelspermissionspremium-featuresdiff --git a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj b/enterprise/backend/src/metabase_enterprise/api_routes/routes.cljindex 6b7e5d7aea6..28bca0d05fd 100644--- a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj+++ b/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj@@ -15,6 +15,7 @@[metabase-enterprise.cloud-proxy.api][metabase-enterprise.content-translation.routes][metabase-enterprise.content-verification.api.routes]+ [metabase-enterprise.data-complexity-score.api][metabase-enterprise.data-studio.api][metabase-enterprise.database-replication.api :as database-replication.api][metabase-enterprise.database-routing.api]@@ -99,6 +100,9 @@"/content-translation" (premium-handler metabase-enterprise.content-translation.routes/routes :content-translation)"/cloud-add-ons" metabase-enterprise.cloud-add-ons.api/routes"/cloud-proxy" metabase-enterprise.cloud-proxy.api/routes+ ;; No premium-handler gate yet — we haven't settled on the feature flag name or final API shape.+ ;; Endpoint is superuser-only so it's not exposed to regular users in the meantime.+ "/data-complexity-score" metabase-enterprise.data-complexity-score.api/routes"/data-studio" (premium-handler metabase-enterprise.data-studio.api/routes :library)"/database-replication" (-> database-replication.api/routes ;; database-replication requires all these features.(premium-handler :attached-dwh)diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 798fc2c6574..6e34503ffc0 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -10,6 +10,7 @@[metabase-enterprise.advanced-config.init][metabase-enterprise.audit-app.init][metabase-enterprise.cache.init]+ [metabase-enterprise.data-complexity-score.init][metabase-enterprise.database-replication.init][metabase-enterprise.dependencies.init][metabase-enterprise.gsheets.init]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.cljnew file mode 100644index 00000000000..8ca1d5dbeb4--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj@@ -0,0 +1,84 @@+(ns metabase-enterprise.data-complexity-score.api+ "Admin-only HTTP endpoint exposing the Data Complexity Score."+ (:require+ [metabase-enterprise.data-complexity-score.complexity :as complexity]+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase.api.common :as api]+ [metabase.api.macros :as api.macros]+ [metabase.api.routes.common :refer [+auth]]))++(set! *warn-on-reflection* true)++(def ^:private SubScore+ "Either a computed sub-score (`:measurement` + `:score`) or an uncomputed one (`:error`)."+ [:or+ [:map {:closed true}+ [:measurement number?]+ [:score nat-int?]]+ [:map {:closed true}+ [:measurement nil?]+ [:score nil?]+ [:error string?]]])++(def ^:private Catalog+ "One catalog's total + per-component breakdown.+ `:total` is nil when any sub-score couldn't be computed — failures cascade through aggregates."+ [:map+ [:total [:maybe nat-int?]]+ [:components+ [:map+ [:entity-count SubScore]+ [:name-collisions SubScore]+ [:synonym-pairs SubScore]+ [:field-count SubScore]+ [:repeated-measures SubScore]]]])++(def ^:private EmbeddingModelMeta+ "Identifies the embedding model backing the synonym calculations, so benchmark consumers can pin to it.+ `nil` when semantic search isn't configured on this instance."+ [:maybe+ [:map+ [:provider string?]+ [:model-name string?]]])++(def ^:private ComplexityScoresResponse+ "Full response body for `GET /api/ee/data-complexity-score/complexity`."+ [:map+ [:library Catalog]+ [:universe Catalog]+ [:metabot Catalog]+ [:meta+ [:map+ [:formula-version pos-int?]+ [:synonym-threshold number?]+ [:embedding-model {:optional true} EmbeddingModelMeta]]]])++;; Per-JVM single-flight guard for the /complexity endpoint. Each scoring run walks the entire+;; app-db catalog and emits Snowplow events, so concurrent superuser requests on the same node+;; would just multiply load and noise without producing different results — fast-fail with 409+;; instead. In a clustered deployment the guard is per-node, so up to one pass per node can still+;; run concurrently; we accept that since superuser API traffic is low-volume. The Quartz job+;; already has its own concurrency control (`DisallowConcurrentExecution` + cluster lock for boot+;; emission), so we deliberately don't share this guard with the task path; a daily cron run that+;; coincided with an API call shouldn't be cancelled.+(defonce ^:private ^java.util.concurrent.atomic.AtomicBoolean api-scoring-running?+ (java.util.concurrent.atomic.AtomicBoolean. false))++(api.macros/defendpoint :get "/complexity" :- ComplexityScoresResponse+ "Return the current Data Complexity Score for this instance.+ Superuser-only, expensive, and emits Snowplow events for benchmark consumers. Concurrent+ requests on the same JVM fast-fail with HTTP 409 — a scoring pass walks the full app-db+ catalog and one in-flight run per node is enough. The guard is per-JVM, so in a clustered+ deployment each node can still run one pass concurrently."+ [_route _query _body]+ (api/check-superuser)+ (when-not (.compareAndSet api-scoring-running? false true)+ (throw (ex-info "Data Complexity Score calculation already in progress" {:status-code 409})))+ (try+ (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))+ (finally+ (.set api-scoring-running? false))))++(def ^{:arglists '([request respond raise])} routes+ "`/api/ee/data-complexity-score` routes."+ (api.macros/ns-handler *ns* +auth))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.cljnew file mode 100644index 00000000000..ad2cf22d317--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj@@ -0,0 +1,482 @@+(ns metabase-enterprise.data-complexity-score.complexity+ "Computes the Data Complexity Score for the semantic layer across three catalogs:+ :library — the curated subset (Cards of type :model and :metric)+ :universe — everything (library entities + all active physical tables)+ :metabot — what the internal Metabot can surface, narrowed by a caller-supplied scope."+ (:require+ [clojure.pprint :as pprint]+ [clojure.string :as str]+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]+ [metabase-enterprise.semantic-search.core :as semantic-search]+ [metabase.analytics.core :as analytics]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.audit-app.core :as audit]+ [metabase.collections.core :as collections]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def formula-version+ "Bump when the scoring formula changes in a way that would break historical comparisons."+ 1)++(def weights+ "Per-axis weights applied to raw measurements. Public because they're part of the scoring+ fingerprint — a tuning change must force a re-score and be visible to Snowplow consumers+ without bumping `formula-version`."+ {:entity 10+ :name-collision 100+ :synonym-pair 50+ :field 1+ :repeated-measure 2})++(def ^:private component->group+ "Thematic parent per sub-component — drives the `<group>.total` + `<group>.<component>` rollup in+ emitted keys so operators can tell `size` from `ambiguity` without SQL.+ Must cover every key produced by [[score-catalog]] or the missing ones emit as `nil.<component>`."+ {:entity-count :size+ :field-count :size+ :name-collisions :ambiguity+ :synonym-pairs :ambiguity+ :repeated-measures :ambiguity})++(def synonym-similarity-threshold+ "Cosine-similarity cutoff for flagging two names as synonyms.+ Higher than semantic-search's retrieval cutoff (0.30) because scoring needs precision, not recall.+ See https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb for how this+ value was chosen."+ 0.90)++;;; ----------------------------------- enumeration -----------------------------------+;;;+(defn- table-field-counts+ "Return `{table-id field-count}` for active fields on the given `table-ids`. Single group-by query."+ [table-ids]+ (if (empty? table-ids)+ {}+ (into {}+ (map (juxt :table_id :field_count))+ (t2/query {:select [:table_id [:%count.* :field_count]]+ :from [:metabase_field]+ :where [:and+ [:= :active true]+ [:in :table_id table-ids]]+ :group-by [:table_id]}))))++(defn- table-measure-names+ "Return `{table-id [measure-name ...]}` for non-archived Measures on the given `table-ids`. A+ measure is a named MBQL aggregation attached to a Table — see [[metabase.measures.models.measure]]."+ [table-ids]+ (if (empty? table-ids)+ {}+ (u/group-by :table_id :name+ (t2/select [:model/Measure :table_id :name]+ :archived false+ :table_id [:in table-ids]))))++(defn- ->card-entity+ "Shape a Card row into an entity map for scoring. Cards don't contribute to `:field-count` in+ v1 — the proposal's +1-per-field rule is about physical Table fields, not Card result columns —+ so we can skip the fat `result_metadata` column entirely. Measures are a separate first-class+ model tied to Tables, not Cards, so Cards also don't contribute to `:measure-names`."+ [{:keys [id name type]}]+ {:id id+ :name name+ :kind (keyword type)+ :field-count 0+ :measure-names []})++(defn- ->table-entity [field-counts measure-names {:keys [id name]}]+ {:id id+ :name name+ :kind :table+ :field-count (get field-counts id 0)+ :measure-names (get measure-names id [])})++(defn- library-collection-ids+ "Set of collection IDs that make up the Library (root + descendants). Empty when the instance has no Library yet."+ []+ (into #{}+ (when-let [root (collections/library-collection)]+ ;; This is cheaper and less brittle than referencing the collection type constants for every entity type.+ (cons (:id root) (collections/descendant-ids root)))))++(defn- metabot-collection-scope-ids+ "Set of collection IDs the internal Metabot can see — its `collection_id` plus descendants.+ nil when no collection scope is configured (Metabot retrieves from everywhere). If the+ collection row can't be loaded (stale/invalid id) we still return a singleton set with the+ raw id so the catalog matches `metabot-metrics-and-models-query`, which filters on the raw+ `collection_id` and returns an empty result rather than dropping the filter."+ [collection-id]+ (when collection-id+ (into #{collection-id}+ (when-let [root (t2/select-one :model/Collection :id collection-id)]+ (collections/descendant-ids root)))))++(defn- verified-card-id-set+ "Set of Card ids whose most-recent moderation review is `verified`. Called only when+ `metabot-scope` requests verified-only filtering — avoids a `moderation_review` join on the+ universe Card select by pushing the check into a small auxiliary lookup."+ []+ (t2/select-fn-set :moderated_item_id :model/ModerationReview+ :moderated_item_type "card"+ :most_recent true+ :status "verified"))++(defn- routed-child-database-id-set+ "Set of database ids whose `router_database_id` is non-nil — the routed child databases whose+ tables Metabot/search hide. Tables with `:db_id` in this set are excluded from the `:metabot`+ catalog, mirroring the table-visibility rule in `metabase.warehouse-schema.models.table`."+ []+ (t2/select-fn-set :id :model/Database :router_database_id [:not= nil]))++(defn- pick-by-row+ "Filter `entities` by `row-pred` applied to the correspondingly-indexed `rows`. Preserves+ reference identity on the kept entity maps so library/metabot vectors share map instances+ with the universe vector rather than allocating fresh ones.+ A nil `row-pred` short-circuits to `[]` — callers whose filter is statically empty (e.g. no+ Library collections on this instance) pass nil to skip enumeration entirely."+ [row-pred rows entities]+ (if row-pred+ (into []+ (keep-indexed (fn [i row] (when (row-pred row) (nth entities i))))+ rows)+ []))++(defn- enumerate-catalogs+ "One-pass enumeration of all three scoring catalogs. Returns+ `{:library [...] :universe [...] :metabot [...]}` where entity maps are shared *by reference*+ across catalogs — a Card or Table that appears in more than one catalog is one map in memory,+ not three.++ An earlier revision fetched the three catalogs separately, which duplicated DB work up to 3×+ per scoring run — most expensively on [[table-field-counts]] and [[table-measure-names]],+ each a `GROUP BY` scan over `metabase_field` / `measure`. We now fetch the universe superset+ once and derive the `:library` and `:metabot` subsets by in-memory filter, which collapses 6+ DB round-trips + 2 auxiliaries into 4 + up-to-2 and lets the aggregates run once each.++ `metabot-scope` is `{:verified-only? <bool> :collection-id <nil|Long>}` describing how the+ internal Metabot narrows its Cards further — it only adds filters, never widens. The caller+ owns the scope decision (premium-feature gate + Metabot row lookup); this namespace does not+ read settings, premium-feature gates, or Metabot rows directly."+ [{:keys [verified-only? collection-id]}]+ (let [library-cids (library-collection-ids)+ metabot-cids (metabot-collection-scope-ids collection-id)+ verified-ids (when verified-only? (verified-card-id-set))+ routed-db-ids (routed-child-database-id-set)+ ;; Extra columns (`:collection_id`, `:is_published`, `:visibility_type`, `:db_id`) are+ ;; selected purely to drive the in-memory library/metabot derivations below — they're+ ;; ignored by `->card-entity` / `->table-entity`. `:card_schema` is required by+ ;; `:model/Card`'s post-select hooks even when we don't otherwise use it.+ universe-cards (t2/select [:model/Card :id :name :type :collection_id :card_schema]+ :type [:in ["metric" "model"]]+ :archived false+ :database_id [:not= audit/audit-db-id])+ universe-tables (t2/select [:model/Table :id :name :collection_id :is_published+ :visibility_type :db_id]+ :active true+ :db_id [:not= audit/audit-db-id])+ field-counts (table-field-counts (mapv :id universe-tables))+ measure-names (table-measure-names (mapv :id universe-tables))+ card-entities (mapv ->card-entity universe-cards)+ table-entities (mapv #(->table-entity field-counts measure-names %) universe-tables)+ in-library-card? (when (seq library-cids)+ (fn [{:keys [collection_id]}]+ (contains? library-cids collection_id)))+ in-library-table? (when (seq library-cids)+ (fn [{:keys [collection_id is_published]}]+ (and is_published+ (contains? library-cids collection_id))))+ in-metabot-card? (when (and (or (nil? metabot-cids) (seq metabot-cids))+ (or (not verified-only?) (seq verified-ids)))+ (fn [{:keys [collection_id id]}]+ (and (or (nil? metabot-cids)+ (contains? metabot-cids collection_id))+ (or (not verified-only?)+ (contains? verified-ids id)))))+ in-metabot-table? (fn [{:keys [visibility_type db_id]}]+ (and (nil? visibility_type)+ (not (contains? routed-db-ids db_id))))]+ {:universe (into card-entities table-entities)+ :library (into (pick-by-row in-library-card? universe-cards card-entities)+ (pick-by-row in-library-table? universe-tables table-entities))+ :metabot (into (pick-by-row in-metabot-card? universe-cards card-entities)+ (pick-by-row in-metabot-table? universe-tables table-entities))}))++;;; ------------------------------------- scoring -------------------------------------++(defn- component-score+ "Sub-score map: raw `:measurement` (double — future-proofs non-integer axes like density) and+ weighted `:score` using the weight at `weight-key`."+ [weight-key n]+ {:measurement (double n)+ :score (* n (get weights weight-key))})++(defn- repeated-names+ "Count of name occurrences past the first (normalized for comparison). Single pass, no+ intermediate frequency map. `raw-names` may contain nils — they're skipped."+ [raw-names]+ (second+ (reduce (fn [[seen repeats] raw-name]+ (if-let [n-name (embedders/normalize-name raw-name)]+ (if (contains? seen n-name)+ [seen (inc repeats)]+ [(conj seen n-name) repeats])+ [seen repeats]))+ [#{} 0]+ raw-names)))++(defn- score-entity-count [entities]+ (component-score :entity (count entities)))++(defn- score-name-collisions [entities]+ (component-score :name-collision (repeated-names (map :name entities))))++(defn- score-field-count [entities]+ (component-score :field (reduce + (keep :field-count entities))))++(defn- score-repeated-measures [entities]+ (component-score :repeated-measure (repeated-names (mapcat :measure-names entities))))++(defn- dot ^double [^floats a ^floats b]+ (let [len (alength a)]+ (loop [i 0 acc 0.0]+ (if (< i len)+ (recur (inc i) (+ acc (* (aget a i) (aget b i))))+ acc))))++(defn- synonym-pair?+ "True when two vectors' cosine similarity is ≥ sqrt(`threshold-sq`).+ Uses the squared form of the inequality — `(a·b)² ≥ t² · ‖a‖² · ‖b‖²` when `a·b ≥ 0`, avoiding+ two `Math/sqrt` calls a direct cosine-similarity computation would need.+ The non-negative guard keeps it sound (squaring a negative `a·b` would flip the inequality).++ `norms-product` is `‖a‖² · ‖b‖²` precomputed by the caller; folding it into one arg keeps us+ within Clojure's 4-argument cap for primitive-typed `defn`s."+ [^floats a ^floats b ^double norms-product ^double threshold-sq]+ (and (pos? norms-product)+ (let [dot-ab (dot a b)]+ (and (>= dot-ab 0.0)+ (>= (* dot-ab dot-ab) (* threshold-sq norms-product))))))++(defn- synonym-pair-count+ "Count of vector pairs whose cosine similarity is ≥ `threshold`. Walks the upper triangle of the+ N×N pair matrix; each vector's `‖v‖²` is precomputed once and reused across every comparison it+ participates in.++ TODO: this is O(N²) in the distinct-name count. Fine while the signal source is the shared+ search-index (bounded by what the indexer has seen), but once we introduce a dedicated name-only+ embedder this should revisit — either as a chunked `M·Mᵀ` via Neanderthal/dtype-next or as a+ dedicated pgvector name-index doing the join in SQL."+ [embeddings threshold]+ (let [n (count embeddings)+ threshold-sq (* threshold threshold)+ ^doubles norms-sq (double-array n)]+ (dotimes [i n]+ (let [v ^floats (embeddings i)]+ (aset norms-sq ^long i (dot v v))))+ (count+ (for [i (range n)+ j (range (inc ^long i) n)+ :when (synonym-pair? (embeddings i) (embeddings j)+ (* (aget norms-sq i) (aget norms-sq j))+ threshold-sq)]+ :value-doesnt-matter))))++(defn- score-synonym-pairs+ "Compute the synonym sub-score for `entities` using `embedder`. On embedder failure, returns nil+ `:score`/`:measurement` plus an `:error` string so the failure cascades through aggregates+ instead of being mistaken for a real zero. A nil `embedder` produces an empty lookup and scores+ a real zero."+ [entities embedder]+ (try+ (let [name->vec (or (and embedder (embedder entities)) {})+ ;; We need to materialize all these vectors in a clojure vec for efficient pairwise similarity checks.+ known-vectors (into []+ (comp (keep (comp embedders/normalize-name :name))+ (distinct)+ (keep name->vec))+ entities)+ pairs (synonym-pair-count known-vectors synonym-similarity-threshold)]+ (component-score :synonym-pair pairs))+ (catch Throwable t+ (log/warn t "Complexity score: synonym detection failed; cascading nil through aggregates")+ (let [msg (some-> (.getMessage t) str/trim)+ err (if (str/blank? msg)+ (or (some-> (class t) .getName) "synonym detection failed")+ msg)]+ {:measurement nil :score nil :error err}))))++(defn- nil-safe-sum+ "Sum `xs` (numbers and/or nils). Returns nil if any element is nil — used to cascade an+ uncomputed sub-score through aggregates instead of silently low-biasing the total with zeros."+ [xs]+ (when (every? some? xs)+ (reduce + xs)))++(defn score-catalog+ "Pure: compute the score breakdown for a catalog given its `entities` and an optional `embedder`."+ [entities embedder]+ (let [components {:entity-count (score-entity-count entities)+ :name-collisions (score-name-collisions entities)+ :synonym-pairs (score-synonym-pairs entities embedder)+ :field-count (score-field-count entities)+ :repeated-measures (score-repeated-measures entities)}]+ {:total (nil-safe-sum (map (comp :score val) components))+ :components components}))++;;; ----------------------------------- public API ------------------------------------++(defn- log-scores!+ "Log the result at :info so operators see the score in app logs even when Snowplow is off."+ [result]+ (log/info (str "Semantic complexity score:\n"+ ;; `pprint` goes through `with-out-str`, not `*out*`, so the "use metabase.util.log" lint is n/a.+ #_{:clj-kondo/ignore [:discouraged-var]}+ (with-out-str (pprint/pprint result)))))++(defn- snake ^String [x]+ (str/replace (name x) "-" "_"))++(defn- dotted-key+ "Join `parts` with `.` after snake-casing each. `(dotted-key :size :entity-count)` → `\"size.entity_count\"`."+ [& parts]+ (str/join "." (map snake parts)))++(defn- parameters-map+ "Sorted-map of scoring inputs likely to evolve, published as a JSON object on each event.+ String keys (top-level and nested) so they round-trip unchanged — Snowplow's `payload` only+ snake-cases top-level keys, and Cheshire would serialize nested keyword keys with their leading+ colon. Excludes `formula_version` — that stays top-level as the primary cross-version filter."+ [{:keys [synonym-threshold embedding-model weights]}]+ (cond-> (sorted-map "synonym_threshold" synonym-threshold+ "weights" (into (sorted-map)+ (map (fn [[k v]] [(snake k) v]))+ weights))+ embedding-model (assoc "embedding_model_provider" (:provider embedding-model)+ "embedding_model_name" (:model-name embedding-model))))++(def ^:private max-error-length+ "Matches the Snowplow schema's `error` maxLength — a pathological exception message+ must not fail validation and drop the whole event."+ 1024)++(defn- truncate-error [s]+ (cond-> s (< max-error-length (count s)) (subs 0 max-error-length)))++(defn- emit-snowplow!+ "Submits Snowplow events for every score, every group aggregation, and the grand total.+ Returns true when they are all successfully delivered.+ Returns false when tracking is disabled or any emission failed."+ [{:keys [library universe metabot meta]}]+ (let [base {:event :data_complexity_scoring+ :formula_version (:formula-version meta)+ :parameters (parameters-map meta)}+ events (for [[catalog result] [[:library library] [:universe universe] [:metabot metabot]]+ event (concat (for [[component sub] (:components result)]+ ;; leaf component+ (cond-> (assoc base+ :catalog catalog+ :key (dotted-key (component->group component) component)+ :score (:score sub)+ :measurement (:measurement sub))+ (:error sub) (assoc :error (truncate-error (:error sub)))))+ (for [[group entries] (group-by #(component->group (key %)) (:components result))]+ ;; group total+ (assoc base+ :catalog catalog+ :key (dotted-key group :total)+ :score (nil-safe-sum (map (comp :score val) entries))))+ ;; grand total+ [(assoc base+ :catalog catalog+ :key (dotted-key :total)+ :score (:total result))])]+ event)]+ ;; No short-circuiting - even if they are failures, attempt the rest.+ (reduce (fn [all-ok? event]+ (and (analytics/track-event! :snowplow/data_complexity event) all-ok?))+ ;; Since events cannot be empty, we don't need to worry about returning a vacuous true.+ true+ events)))++(defn- time-phase!+ "Run `f`, record duration on the per-phase histogram labelled by `stage` and `catalog`, return its value."+ [stage catalog f]+ (let [timer (u/start-timer)]+ (try+ (f)+ (finally+ (prometheus/observe! :metabase-data-complexity/phase-duration-ms+ {:stage stage :catalog catalog}+ (u/since-ms timer))))))++(defn complexity-scores+ "Compute the complexity score for the `:library`, `:universe`, and `:metabot` catalogs of this+ Metabase instance. Returns a map of the shape:++ {:library {:total n :components {...}}+ :universe {:total n :components {...}}+ :metabot {:total n :components {...}}+ :meta {:formula-version 1+ :synonym-threshold 0.90+ :embedding-model {...}}}++ Options:+ `:embedder` — overrides the synonym-axis embedder (defaults to+ [[metabase-enterprise.semantic-search.core/search-index-embedder]]); pass `nil` to disable+ synonym scoring.+ `:metabot-scope` — a `{:verified-only? <bool> :collection-id <nil|Long>}` map describing how+ the internal Metabot filters Cards. `:metabot` is always scored separately from `:universe`+ because Metabot/search table visibility (hidden tables, routed databases) already diverges+ from the raw `:universe` set; the scope only narrows Cards further. The caller owns the+ scope decision (premium-feature gate + Metabot row lookup) so this namespace stays free of+ settings/feature/Metabot-row reads."+ [& {:keys [embedder metabot-scope] :as opts}]+ ;;; NOTE: we fully materialize a vector off all entities, along with one of those in the library, rather than+ ;;; returning reducibles. For very large instances that holds a non-trivial slim-entity list in memory+ ;;; (name, kind, field-count, measure-names — no fat columns like `result_metadata`),+ ;;; but each catalog is consumed by FIVE sub-score functions that each walk the collection, so making this+ ;;; reducible would re-query the app-db five times per scoring call — a worse tradeoff than the bounded memory+ ;;; we currently will currently consume (provided we have that memory).+ (let [total-timer (u/start-timer)]+ (try+ (let [embedder (if (contains? opts :embedder)+ embedder+ semantic-search/search-index-embedder)+ model-meta (when (= embedder semantic-search/search-index-embedder)+ (semantic-search/active-embedding-model))+ {:keys [library universe metabot]}+ ;; Single enumerate phase — see [[enumerate-catalogs]]. Library ⊆ universe and+ ;; metabot ⊆ universe, so fetching each catalog separately duplicated DB work; the+ ;; catalog label `"all"` on the enumerate stage reflects that one pass covers all+ ;; three. Per-catalog timing only applies to the pure scoring step.+ (time-phase! "enumerate" "all" #(enumerate-catalogs metabot-scope))+ universe-score (time-phase! "score" "universe" #(score-catalog universe embedder))+ library-score (time-phase! "score" "library" #(score-catalog library embedder))+ metabot-score (time-phase! "score" "metabot" #(score-catalog metabot embedder))+ result {:library library-score+ :universe universe-score+ :metabot metabot-score+ :meta (cond-> {:formula-version formula-version+ :synonym-threshold synonym-similarity-threshold}+ model-meta (assoc :embedding-model model-meta))}]+ (log-scores! result)+ (let [published? (try+ (emit-snowplow! result)+ (catch Throwable t+ (log/warn t "Failed to publish complexity score to Snowplow")+ false))]+ ;; `emit-snowplow!` returns true only when every event reached the tracker (false when+ ;; Snowplow is disabled or any emission failed) — scheduler/boot callers gate+ ;; `data-complexity-scoring-last-fingerprint` on this so a disabled collector or any+ ;; partial failure doesn't silently mark the fingerprint as published.+ (with-meta result {::snowplow-published? published?})))+ (finally+ (prometheus/observe! :metabase-data-complexity/scoring-duration-ms+ (u/since-ms total-timer))))))++(comment+ (complexity-scores))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.cljnew file mode 100644index 00000000000..ce2cdfc04b0--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj@@ -0,0 +1,24 @@+(ns metabase-enterprise.data-complexity-score.complexity-embedders+ "Pluggable embedding sources for the complexity score's synonym axis.+ An embedder takes entities `{:id :name :kind}` and returns `{normalized-name -> ^floats vector}`,+ omitting entities without a known vector."+ (:require+ [clojure.string :as str]+ [metabase.util :as u]))++(set! *warn-on-reflection* true)++(defn normalize-name+ "Canonical form used for name-based lookups and comparisons. nil-safe."+ [s]+ (some-> s str/trim u/lower-case-en))++(defn fn-embedder+ "Build an embedder that delegates to a plain `(name-embed-fn names) -> [vectors]` function.+ Distinct normalized names are passed in; the returned vectors are zipped back by position.+ Names whose `name-embed-fn` returns nil are omitted from the result map."+ [name-embed-fn]+ (fn embed [entities]+ (let [names (->> entities (keep (comp normalize-name :name)) distinct vec)+ vectors (when (seq names) (vec (name-embed-fn names)))]+ (into {} (filter val) (zipmap names vectors)))))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.cljnew file mode 100644index 00000000000..2c07d18763c--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/init.clj@@ -0,0 +1,14 @@+(ns metabase-enterprise.data-complexity-score.init+ "Loader + startup wiring for data-complexity-score tasks.+ Ensures boot-time Data Complexity Score publishing runs regardless of `MB_DISABLE_SCHEDULER`."+ (:require+ [metabase-enterprise.data-complexity-score.settings]+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]+ [metabase.startup.core :as startup]+ [metabase.util.quick-task :as quick-task]))++(set! *warn-on-reflection* true)++(defmethod startup/def-startup-logic! ::EmitComplexityScoreIfStale [_]+ ;; Background thread — keeps boot latency flat even when scoring is slow.+ (quick-task/submit-task! task.complexity-score/maybe-emit-boot-score!))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.cljnew file mode 100644index 00000000000..0a67d6af20c--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/metabot_scope.clj@@ -0,0 +1,15 @@+(ns metabase-enterprise.data-complexity-score.metabot-scope+ "Resolves the internal Metabot's retrieval scope for the `:metabot` catalog."+ (:require+ [metabase.metabot.config :as metabot.config]+ [metabase.premium-features.core :as premium-features]+ [toucan2.core :as t2]))++(defn internal-metabot-scope+ "`{:verified-only? <bool> :collection-id <nil|Long>}` matching what Metabot's Card query would apply."+ []+ (let [entity-id (get-in metabot.config/metabot-config [metabot.config/internal-metabot-id :entity-id])+ metabot (t2/select-one :model/Metabot :entity_id entity-id)]+ {:verified-only? (and (boolean (:use_verified_content metabot))+ (premium-features/has-feature? :content-verification))+ :collection-id (:collection_id metabot)}))diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.cljnew file mode 100644index 00000000000..e1db149d62f--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj@@ -0,0 +1,32 @@+(ns metabase-enterprise.data-complexity-score.settings+ "Settings for the data-complexity-score module."+ (:require+ [metabase.settings.core :refer [defsetting]]+ [metabase.util.i18n :refer [deferred-tru]]))++(defsetting data-complexity-scoring-enabled+ (deferred-tru "Run the scheduled Data Complexity Score job (one node per cluster, daily).")+ :encryption :no+ :visibility :admin+ :default false+ :type :boolean+ :export? false+ :doc false)++(defsetting data-complexity-scoring-last-fingerprint+ (deferred-tru "Internal bookkeeping: Fingerprint of last successful run, to prevent needless re-calculations.")+ :encryption :no+ :visibility :internal+ :default ""+ :type :string+ :export? false+ :doc false)++(defsetting data-complexity-scoring-claim+ (deferred-tru "Internal bookkeeping: EDN metadata with TTL for in-progress run, to skip duplicate work.")+ :encryption :no+ :visibility :internal+ :default ""+ :type :string+ :export? false+ :doc false)diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.cljnew file mode 100644index 00000000000..4f75dd4461b--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj@@ -0,0 +1,198 @@+(ns metabase-enterprise.data-complexity-score.task.complexity-score+ "Daily Quartz job that computes and publishes the Data Complexity Score."+ (:require+ [clojure.edn :as edn]+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase-enterprise.data-complexity-score.complexity :as complexity]+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase-enterprise.data-complexity-score.settings :as settings]+ [metabase-enterprise.semantic-search.core :as semantic-search]+ [metabase.app-db.cluster-lock :as cluster-lock]+ [metabase.config.core :as config]+ [metabase.task.core :as task]+ [metabase.util.log :as log])+ (:import+ (org.quartz DisallowConcurrentExecution)))++(set! *warn-on-reflection* true)++(def ^:private job-key (jobs/key "metabase.task.data-complexity-score.job"))+(def ^:private trigger-key (triggers/key "metabase.task.data-complexity-score.trigger"))++(defn- current-fingerprint+ "String capturing everything that changes the meaning of an emitted score — mirror of the Snowplow+ `formula_version` + `parameters` fields. Includes `weights` so re-tuning forces a re-score+ without bumping `formula-version`; only structural changes to the scoring algorithm need that."+ []+ (let [embedding-model (semantic-search/active-embedding-model)]+ (pr-str (into (sorted-map)+ (cond-> {:formula-version complexity/formula-version+ :synonym-threshold complexity/synonym-similarity-threshold+ :weights complexity/weights}+ embedding-model (assoc :embedding-model embedding-model))))))++(defn- run-scoring!+ "One scoring pass. Gated by [[settings/data-complexity-scoring-enabled]] so admins can silence+ scoring without unscheduling the job.++ `claim-fingerprint` is the fingerprint carried on the scoring claim that authorized this run.+ Using it (rather than re-sampling [[current-fingerprint]] at commit time) means a config change+ mid-run cannot make us stamp a fingerprint onto `last-fingerprint` that we didn't actually score.++ Returns the score result (with `::complexity/snowplow-published?` metadata) when scoring ran, or+ nil when skipped / threw. Only a confirmed Snowplow publish advances `last-fingerprint` — any+ other outcome leaves it untouched so the next boot or cron retries and telemetry doesn't+ silently stall behind a transient publish failure."+ [claim-fingerprint]+ (if (settings/data-complexity-scoring-enabled)+ (try+ (let [result (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))]+ (if (::complexity/snowplow-published? (meta result))+ (settings/data-complexity-scoring-last-fingerprint! claim-fingerprint)+ (log/warn "Data Complexity Score: Snowplow publish failed; leaving fingerprint unchanged so the next boot or cron retries"))+ result)+ (catch Throwable t+ (log/warn t "Data Complexity Score run failed")))+ (log/debug "Data Complexity Score run skipped — data-complexity-scoring-enabled is off")))++;; Long enough that any realistic scoring run finishes well inside it, short enough that a crashed+;; claimant unblocks the next tick's retry without operator intervention.+(def ^:private scoring-claim-ttl-ms (* 30 60 1000))++(defn- now-ms [] (System/currentTimeMillis))++(defn- parse-scoring-claim+ "Parse the persisted claim string into `{:fingerprint <str> :claimed-at <long> :owner <str>}` or+ nil when the setting is empty/garbled."+ [s]+ (when (and s (seq s))+ (try+ (let [v (edn/read-string s)]+ (when (map? v) v))+ (catch Throwable _ nil))))++(defn- scoring-claim-active?+ "True when a persisted claim for the same fingerprint we'd score now is still within the TTL —+ meaning another path (sibling boot, cron tick) is already computing and publishing for it, so+ this caller should skip."+ [claim current-fp]+ (boolean+ (and claim+ (= current-fp (:fingerprint claim))+ (when-let [ts (:claimed-at claim)]+ (and (int? ts)+ (< (- (now-ms) (long ts)) scoring-claim-ttl-ms))))))++(defn- claim-scoring-run!+ "Cluster-safe claim on one Data Complexity Score run — shared by the cron job and the boot-time+ startup hook so the two paths cannot compute and publish in parallel. Returns the claim map+ (including a unique `:owner` token) when this caller wins the right to run, or nil when another+ node/path already holds an active claim for the current fingerprint — or, when+ `:require-fingerprint-change?` is true, when the live fingerprint still matches the last+ successfully-published one (boot's extra gate so restarts at an unchanged config don't re-emit).++ The returned claim must be passed back to [[release-scoring-claim!]] so the release is a+ compare-and-clear — a sibling that took over after a TTL-based expiry keeps its own claim.++ We do not advance [[settings/data-complexity-scoring-last-fingerprint]] here — only `run-scoring!`+ does, after confirmed Snowplow publish. Instead we write a separate+ [[settings/data-complexity-scoring-claim]] marker (fingerprint + timestamp + owner) so that:++ - other nodes/paths acquiring the lock next see an active claim and skip (no duplicate scoring);+ - a scoring/publish failure or crash mid-run leaves the success fingerprint untouched, so the+ next boot or cron retries instead of treating the fingerprint as up-to-date;+ - the TTL on `:claimed-at` means even a hard JVM crash doesn't permanently suppress scoring;+ - the `:owner` token lets a slow claimant distinguish its own claim from a sibling's takeover.++ Reads inside the lock bypass the in-process settings cache via `config/*disable-setting-cache*`+ so values freshly committed by sibling nodes are always visible — the cache is only refreshed+ periodically and without bypass two nodes could both pass the check and both score."+ [{:keys [require-fingerprint-change?]}]+ (cluster-lock/with-cluster-lock {:lock ::complexity-score-run+ :timeout-seconds 10}+ (binding [config/*disable-setting-cache* true]+ (let [current (current-fingerprint)]+ (when (and (settings/data-complexity-scoring-enabled)+ (or (not require-fingerprint-change?)+ (not= current (settings/data-complexity-scoring-last-fingerprint)))+ (not (scoring-claim-active?+ (parse-scoring-claim (settings/data-complexity-scoring-claim))+ current)))+ (let [claim {:fingerprint current+ :claimed-at (now-ms)+ :owner (str (random-uuid))}]+ (settings/data-complexity-scoring-claim! (pr-str claim))+ claim))))))++(defn- release-scoring-claim!+ "Best-effort compare-and-clear of the caller's scoring claim. Only clears the persisted value+ when its `:owner` still matches `claim` — protecting against the case where our run outlived the+ TTL and a sibling legitimately took over the claim in the meantime. Failure is non-fatal: the TTL+ on the claim timestamp ensures the next boot/cron can retry either way."+ [claim]+ (try+ (cluster-lock/with-cluster-lock {:lock ::complexity-score-run+ :timeout-seconds 10}+ (binding [config/*disable-setting-cache* true]+ (let [persisted (parse-scoring-claim (settings/data-complexity-scoring-claim))]+ (if (and persisted (= (:owner persisted) (:owner claim)))+ (settings/data-complexity-scoring-claim! "")+ (log/info "Data Complexity Score: skipping claim release — persisted claim no longer belongs to this node (sibling takeover after TTL)")))))+ (catch Throwable t+ (log/warn t "Data Complexity Score: failed to clear scoring claim; it will expire via TTL"))))++(defn- with-scoring-claim!+ "Acquire a scoring claim, invoke `f` with the claim's fingerprint, then release the claim via+ compare-and-clear. Returns nil when the claim couldn't be acquired (another path is scoring, or+ the fingerprint-change gate didn't fire). `opts` pass through to [[claim-scoring-run!]]."+ [opts f]+ (when-let [claim (claim-scoring-run! opts)]+ (try+ (f (:fingerprint claim))+ (finally+ (release-scoring-claim! claim)))))++(task/defjob ^{DisallowConcurrentExecution true+ :doc "Compute and publish the Data Complexity Score."}+ DataComplexityScoring [_ctx]+ (with-scoring-claim! {} run-scoring!))++(defn maybe-emit-boot-score!+ "Run one scoring pass at boot when the persisted fingerprint lags the live config. Shares the+ cluster-wide scoring claim with the cron job (see [[claim-scoring-run!]]) so a restart that+ lands on top of a 03:17 UTC cron tick can't double-publish. Runs regardless of Quartz state so+ operators still get a score on nodes with `MB_DISABLE_SCHEDULER=true` or a failed scheduler init.+ Intended to be called from a startup hook on a background thread — see+ `metabase-enterprise.data-complexity-score.init`.++ Success advances the last-successful fingerprint inside `run-scoring!`; any other outcome (skip,+ throw, publish failure) leaves it untouched so the next boot or cron retries. The claim is+ released (compare-and-clear on `:owner`) after the run so other paths can proceed without+ waiting for the TTL — unless our run outlived the TTL and another path took over the claim."+ []+ (try+ (with-scoring-claim! {:require-fingerprint-change? true}+ (fn [fp]+ (log/info "Data Complexity Score: fingerprint changed, emitting boot-time score")+ (run-scoring! fp)))+ (catch Throwable t+ (log/warn t "Data Complexity Score: boot-time emission failed"))))++(defmethod task/init! ::DataComplexityScoring [_]+ (let [job (jobs/build+ (jobs/of-type DataComplexityScoring)+ (jobs/store-durably)+ (jobs/with-identity job-key)+ (jobs/with-description "Data Complexity Score — daily telemetry"))+ ;; 03:17 UTC — off-hour to avoid cron-thundering-herd with other Metabase jobs.+ trigger (triggers/build+ (triggers/with-identity trigger-key)+ (triggers/for-job job-key)+ (triggers/with-schedule+ (cron/schedule+ (cron/cron-schedule "0 17 3 * * ? *")+ (cron/in-time-zone (java.util.TimeZone/getTimeZone "UTC"))+ (cron/with-misfire-handling-instruction-fire-and-proceed))))]+ (task/schedule-task! job trigger)))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex d9683ca4fbe..7add9418d76 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -4,6 +4,7 @@[clojure.string :as str][medley.core :as m][metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedders][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]@@ -15,8 +16,14 @@[metabase.tracing.core :as tracing][metabase.util :as u][metabase.util.log :as log]+ [potemkin :as p][toucan2.realize :as t2.realize]))+(p/import-vars+ [metabase-enterprise.semantic-search.embedders+ active-embedding-model+ search-index-embedder])+(defn- fallback-engine"Find the highest priority search engine available for fallback."[]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.cljnew file mode 100644index 00000000000..6558f6e49ca--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj@@ -0,0 +1,174 @@+(ns metabase-enterprise.semantic-search.embedders+ "Embedders that expose the pgvector semantic-search index as a name → vector lookup.+ An embedder takes entity maps `{:id :name :kind}` and returns `{normalized-name → ^floats vector}`,+ omitting entities not in the index."+ (:require+ [clojure.string :as str]+ [honey.sql :as sql]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase.metabot.core :as metabot]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]))++(set! *warn-on-reflection* true)++(def ^:private fetch-batch-size+ "Maximum number of `(model, model_id)` pairs per SQL query when reading from the pgvector index.+ Keeps the generated OR predicate bounded so we don't hit JDBC bind-parameter limits or produce query+ plans that choke on large installs."+ 500)++(def ^:private float-array-class+ "Cached `[F` class so [[parse-pgvector]] doesn't pay a `Class/forName` lookup per row."+ (Class/forName "[F"))++(defn- normalize-name [s]+ (some-> s str/trim u/lower-case-en))++(defn- prefer-new-row?+ "Return true when `new-row` should replace `prior` as the representative for a normalized name.+ `search-index-embedder` keeps exactly one embedding per normalized name, so the choice of winner+ is load-bearing: different rows carry different indexed text (name + description + other search+ fields) and therefore different embeddings, which changes downstream similarity comparisons.+ Priority chain (lowest wins, must be total so the result is stable across DB row order and batch+ boundaries):+ 1. Numeric `:mid` (parsed `model_id`) — a parseable numeric id always beats a nil-parsed one;+ two numerics compare naturally.+ 2. Raw `model_id` string — lexicographic; needed so `\"2\"` vs `\"02\"` (which parse equal) and+ two non-numeric ids still order deterministically.+ 3. `:model` string — lexicographic final tie-break when `model_id` values are identical across+ different model types."+ [{new-mid :mid new-model :model new-mid-str :model_id}+ {prior-mid :mid prior-model :model prior-mid-str :model_id}]+ (let [num-cmp (cond+ (and new-mid prior-mid) (compare (long new-mid) (long prior-mid))+ new-mid -1 ; numeric beats non-numeric+ prior-mid 1+ :else 0)+ cmp (if-not (zero? num-cmp)+ num-cmp+ (let [str-cmp (compare (str new-mid-str) (str prior-mid-str))]+ (if-not (zero? str-cmp)+ str-cmp+ (compare new-model prior-model))))]+ (neg? cmp)))++(defn- parse-pgvector+ "Parse a pgvector string (\"[0.1, 0.2, ...]\") or similar into a float-array."+ [v]+ (cond+ (nil? v) nil+ (instance? float-array-class v) v+ :else+ (let [s (if (string? v) v (str v))+ nums (->> (-> s+ (str/replace "[" "")+ (str/replace "]" "")+ (str/split #","))+ (keep #(let [t (str/trim %)] (when (seq t) (Float/parseFloat t)))))]+ (float-array nums))))++(defn- fetch-batch+ "Query the active pgvector index table for a single batch of `(model, model_id)` pairs. Returns a+ seq of `{:model :model_id :name :embedding}` rows."+ [pgvector table-name pairs]+ (let [where (into [:or]+ (for [[m mid] pairs]+ [:and [:= :model m] [:= :model_id mid]]))+ sql-vec (sql/format {:select [:model :model_id :name :embedding]+ :from [(keyword table-name)]+ :where where}+ {:quoted true})]+ (jdbc/execute! pgvector sql-vec {:builder-fn jdbc.rs/as-unqualified-lower-maps})))++(defn- reduce-batched-rows+ "Fold `rf` over rows fetched from the pgvector index in batches of [[fetch-batch-size]] pairs.+ Never materializes the full row list — each batch is fetched, reduced into `acc`, and discarded+ before the next batch is fetched. Keeps peak memory to `O(batch + reduced-state)` instead of+ `O(total-rows)`, which matters because raw pgvector embedding strings are ~6–18KB each."+ [pgvector table-name pairs rf init]+ (reduce (fn [acc batch]+ (reduce rf acc (fetch-batch pgvector table-name batch)))+ init+ (partition-all fetch-batch-size pairs)))++(defn- try-active-index-state+ "Return the active index state, or nil if unavailable. Never throws."+ []+ (try+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ md (semantic.env/get-index-metadata)]+ (when-let [state (semantic.index-metadata/get-active-index-state pgvector md)]+ {:pgvector pgvector+ :table-name (-> state :index :table-name)+ :model (-> state :index :embedding-model)}))+ (catch Throwable t+ (log/debug t "Semantic-search index not available; search-index embedder will return {}"))))++(defn search-index-embedder+ "Embedder that reads vectors from the active semantic-search pgvector index. The indexed text+ combines name + description + other search fields, so the similarity signal reflects semantic+ overlap rather than name-only synonymy.+ Returns `{}` when the index isn't available (premium feature off, not yet initialized, datasource+ unreachable).++ Pgvector read failures (e.g. the index table exists but the query errors out) propagate so the+ caller can annotate its own error state — silently returning `{}` would look indistinguishable+ from \"no synonym matches\", which would let transient failures underreport complexity with no+ machine-readable signal. See [[metabase-enterprise.data-complexity-score.complexity/score-synonym-pairs]]+ for the wrapping `catch` that converts this into a `:error` field on the sub-score.++ When multiple entities share a normalized name but have different indexed embeddings the row+ chosen by [[prefer-new-row?]] wins, so the result is deterministic across runs regardless of+ batch boundaries.++ Parses embeddings as we fold so raw pgvector strings (6–18 KB each) aren't retained alongside+ the parsed float-arrays. Peak memory is still O(distinct-names) in float-arrays, which will+ matter on very large instances (>~50k entities) — the pairwise comparison downstream is the+ harder scaling problem and both will want to be revisited together (push similarity into+ pgvector SQL, or switch to HNSW approximate-neighbor)."+ [entities]+ (if-let [{:keys [pgvector table-name]} (try-active-index-state)]+ ;; No explicit `distinct` on `pairs`: callers pass entities from per-catalog SQL queries where+ ;; each `(kind, id)` maps to a unique primary key, so duplicates don't arise in practice. The+ ;; downstream `prefer-new-row?` fold already collapses any that slip through, so the only cost+ ;; of a hypothetical duplicate would be a slightly wider SQL `OR` clause.+ (let [pairs (for [{:keys [id kind]} entities+ :let [m (metabot/entity-type->search-model kind)]+ :when m]+ [m (str id)])+ ;; Fold fetch → parse → dedup in one pass. `prefer-new-row?` keeps the winner globally+ ;; (lowest numeric model_id, then model as tie-break) so the result is stable across+ ;; runs regardless of which 500-row batch boundary a duplicate lands on.+ keyed (when (seq pairs)+ (reduce-batched-rows+ pgvector table-name pairs+ (fn [acc {:keys [name model_id model embedding]}]+ (if-let [vec (parse-pgvector embedding)]+ (let [k (normalize-name name)+ entry {:mid (parse-long model_id)+ :model model+ :model_id model_id+ :vec vec}+ prior (get acc k)]+ (if (or (nil? prior) (prefer-new-row? entry prior))+ (assoc acc k entry)+ acc))+ acc))+ {}))]+ (update-vals (or keyed {}) :vec))+ {}))++(defn active-embedding-model+ "Return the embedding model metadata for the *active* search index, not the current configuration.+ Returns nil when the index is unreachable, not yet initialized, or the feature is disabled.+ Use this instead of [[metabase-enterprise.semantic-search.env/get-configured-embedding-model]] when+ you need to know what model the index is *actually* serving — not just what the settings say."+ []+ (when-let [{:keys [model]} (try-active-index-state)]+ (when model+ {:provider (:provider model)+ :model-name (:model-name model)})))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.cljnew file mode 100644index 00000000000..68f5b673bb2--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj@@ -0,0 +1,232 @@+(ns metabase-enterprise.data-complexity-score.api-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.data-complexity-score.api :as api]+ [metabase-enterprise.data-complexity-score.complexity :as complexity]+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase.metabot.config :as metabot.config]+ [metabase.test :as mt]+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent CountDownLatch TimeUnit)))++(set! *warn-on-reflection* true)++(comment api/keep-me)++(def ^:private endpoint "ee/data-complexity-score/complexity")++(defn- internal-metabot-id+ "Primary key of the internal Metabot row — used by the tests that temporarily tweak its+ `use_verified_content`/`collection_id` via `mt/with-temp-vals-in-db`. Calls+ `mt/initialize-if-needed!` so the row is populated by migrations even when this test runs in+ isolation (the endpoint tests piggyback on the web-server init and miss this failure mode)."+ []+ (mt/initialize-if-needed! :db)+ (t2/select-one-pk :model/Metabot+ :entity_id (get-in metabot.config/metabot-config+ [metabot.config/internal-metabot-id :entity-id])))++(deftest complexity-endpoint-requires-superuser-test+ (testing "non-superusers are rejected"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 endpoint)))))++(deftest complexity-endpoint-superuser-gets-consistent-totals-test+ (testing "check invariants not covered by schema"+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)+ measurement (fn [cat k] (get-in resp [cat :components k :measurement]))+ component-score (fn [cat k] (get-in resp [cat :components k :score]))+ ;; NOTE: `:synonym-pairs` is intentionally included here even though it's *theoretically*+ ;; non-monotonic — `score-synonym-pairs` dedupes by normalized name and keeps whichever+ ;; embedding `search-index-embedder` happens to pick for that name, so adding+ ;; universe-only entities that collide on normalized name with a library entity could in+ ;; principle flip which vector wins and drop the pair count below library's. Reviewers+ ;; (human or AI) sometimes want to carve it out on that basis — don't. In every realistic+ ;; configuration (prod, dev, the fixture that backs this endpoint's hermetic path in+ ;; complexity_test.clj) the invariant holds, and asserting it keeps us honest about+ ;; regressions in the common case. If the edge case ever actually trips this, *that* is+ ;; the surprising thing we want to see and we'll deal with it then.+ component-keys [:entity-count :name-collisions :synonym-pairs+ :field-count :repeated-measures]]+ (testing ":total equals the sum of its component :score values"+ (doseq [catalog [:library :universe :metabot]+ :let [{:keys [total components]} (get resp catalog)]]+ (is (= total (reduce + (map :score (vals components))))+ (format "%s :total should equal sum of component :score values" catalog))))+ (testing "universe is a superset of library: every measurement and score ≥ library's"+ (doseq [k component-keys+ getter [measurement component-score]+ :let [lib (getter :library k)+ uni (getter :universe k)]]+ (is (>= uni lib)+ (format "universe %s (%s) should be ≥ library's (%s)" k uni lib))))+ (testing ":synonym-pairs can't exceed the number of distinct-name pairs possible"+ (doseq [catalog [:library :universe :metabot]+ :let [n-entities (measurement catalog :entity-count)+ syn-pairs (measurement catalog :synonym-pairs)+ max-pairs (/ (* n-entities (dec n-entities)) 2)]]+ (is (<= syn-pairs max-pairs)+ (format "%s :synonym-pairs (%s) can't exceed n*(n-1)/2 for n=%s" catalog syn-pairs n-entities)))))))++(deftest complexity-endpoint-metabot-catalog-test+ (testing ":metabot mirrors :universe when neither content-verification nor use_verified_content is active"+ ;; Pin both gates explicitly instead of relying on test-env defaults — the reused-verbatim path+ ;; is only exercised when the scope is empty, and we want this assertion to keep passing even+ ;; if the ambient defaults shift.+ (mt/with-premium-features #{}+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)+ {:use_verified_content false :collection_id nil}+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]+ (is (= (:universe resp) (:metabot resp)))))))+ (testing ":metabot is scored separately when :content-verification + use_verified_content are both active"+ ;; Positive path: verified-only filtering restricts Cards to those with an active verified+ ;; moderation review. We inject a fresh unverified Card so the assertion doesn't depend on+ ;; ambient test-env content — the `:universe` count includes this Card, `:metabot` excludes it.+ (mt/with-premium-features #{:content-verification}+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Card _ {:database_id db-id+ :type :model+ :name "Unverified Only Card"+ :archived false}]+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)+ {:use_verified_content true :collection_id nil}+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]+ (is (< (get-in resp [:metabot :components :entity-count :measurement])+ (get-in resp [:universe :components :entity-count :measurement]))+ ":metabot entity-count must be strictly < :universe when verified-only filters out the injected Card")))))))++(deftest complexity-endpoint-metabot-collection-scope-test+ (testing ":metabot is scoped to the internal Metabot's collection_id subtree (root + descendants)"+ ;; Fixture shape — exercises both halves of `metabot-collection-scope-ids`:+ ;; parent ← Metabot's collection_id; holds a Card directly (catches root-omitted regressions)+ ;; └ child → holds a nested Card (catches descent-dropped regressions)+ ;; sibling → holds the out-of-subtree Card+ ;; empty → no Cards; baseline scope so we can pin *exact* card-count deltas+ ;;+ ;; Tables pass through :metabot unfiltered, so ambient table counts cancel when we take+ ;; differentials against `empty`. That leaves us a clean count of Cards visible under each+ ;; scope, which lets us assert exact expected counts rather than relative inequalities.+ ;; Pin premium features off explicitly so `:verified-only?` can't drift in and confound.+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Collection {parent-id :id} {:name "Metabot Scope Parent"+ :location "/"}+ :model/Collection {child-id :id} {:name "Metabot Scope Child"+ :location (format "/%d/" parent-id)}+ :model/Collection {sibling-id :id} {:name "Metabot Scope Sibling"+ :location "/"}+ :model/Collection {empty-id :id} {:name "Metabot Scope Empty"+ :location "/"}+ :model/Database {db-id :id} {}+ :model/Card _ {:database_id db-id+ :type :model+ :name "Metabot Root Card"+ :archived false+ :collection_id parent-id}+ :model/Card _ {:database_id db-id+ :type :model+ :name "Metabot In-subtree Nested Card"+ :archived false+ :collection_id child-id}+ :model/Card _ {:database_id db-id+ :type :metric+ :name "Metabot Out-of-subtree Card"+ :archived false+ :collection_id sibling-id}]+ (let [counts-with-scope (fn [cid]+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)+ {:use_verified_content false+ :collection_id cid}+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)]+ {:metabot (get-in resp [:metabot :components :entity-count :measurement])+ :universe (get-in resp [:universe :components :entity-count :measurement])})))+ empty-counts (counts-with-scope empty-id)+ parent-counts (counts-with-scope parent-id)+ sibling-counts (counts-with-scope sibling-id)+ empty-count (:metabot empty-counts)+ parent-count (:metabot parent-counts)+ sibling-count (:metabot sibling-counts)]+ (testing "scope=parent counts both the root Card and the nested descendant Card"+ ;; If the root collection id were dropped from `metabot-collection-scope-ids`, the+ ;; root Card would vanish and `parent-count` would land at `empty-count + 1`. If+ ;; descent regressed, the nested Card would vanish and `parent-count` would again+ ;; be `empty-count + 1`. Either regression fails this assertion.+ (is (= (+ empty-count 2) parent-count)+ "scope=parent :metabot must include BOTH parent-card (root) AND child-card (descendant)"))+ (testing "scope=sibling counts only the in-scope Card (out-of-subtree Card is excluded)"+ ;; If the collection-id filter were dropped entirely, `sibling-count` would jump to+ ;; `empty-count + 3` (all three fixture Cards visible), failing this assertion.+ (is (= (inc empty-count) sibling-count)+ "scope=sibling :metabot must include only sibling-card — collection filter must apply"))+ (testing ":universe entity-count is unaffected by Metabot collection scope"+ ;; Guards against a regression where the subtree filter leaks into `:universe`+ ;; scoring. If it did, `:universe` counts would move in lockstep with `:metabot`+ ;; across the three scopes and this assertion would fail.+ (is (= (:universe empty-counts)+ (:universe parent-counts)+ (:universe sibling-counts))+ ":universe must be unscoped regardless of Metabot.collection_id")))))))++(def ^:private empty-catalog+ {:total 0 :components {:entity-count {:measurement 0.0 :score 0}+ :name-collisions {:measurement 0.0 :score 0}+ :synonym-pairs {:measurement 0.0 :score 0}+ :field-count {:measurement 0.0 :score 0}+ :repeated-measures {:measurement 0.0 :score 0}}})++(def ^:private stub-scores+ {:library empty-catalog :universe empty-catalog :metabot empty-catalog+ :meta {:formula-version 1 :synonym-threshold 0.0}})++(deftest ^:sequential complexity-endpoint-rejects-concurrent-requests-test+ (testing "a second concurrent request fast-fails with 409 instead of running a duplicate scoring pass"+ ;; Block the stubbed scoring call on a latch so the second request is guaranteed to land+ ;; while the guard is held. Plain `with-redefs` (not `with-dynamic-fn-redefs`) because+ ;; the dynamic variant binds thread-locally and the futures we spawn here wouldn't see it.+ (let [release-scoring (CountDownLatch. 1)+ scoring-started (CountDownLatch. 1)+ call-count (atom 0)]+ (with-redefs [complexity/complexity-scores+ (fn [& _]+ (swap! call-count inc)+ (.countDown scoring-started)+ (.await release-scoring 10 TimeUnit/SECONDS)+ stub-scores)]+ (let [first-request (future (mt/user-http-request :crowberto :get 200 endpoint))]+ (try+ (is (.await scoring-started 10 TimeUnit/SECONDS)+ "first request must reach the guarded section before we fire the second")+ (testing "concurrent superuser request is rejected with 409"+ (is (= "Data Complexity Score calculation already in progress"+ (mt/user-http-request :crowberto :get 409 endpoint))))+ (finally+ (.countDown release-scoring)+ ;; Drain the in-flight request so the guard is released before the next test+ ;; (failed assertions above shouldn't leak a stuck scoring call into other tests).+ (deref first-request 10000 ::timeout))))+ (testing "only the first request actually ran scoring; the second short-circuited"+ (is (= 1 @call-count)))+ (testing "guard is released after the in-flight request finishes — a follow-up request succeeds"+ (mt/user-http-request :crowberto :get 200 endpoint)+ (is (= 2 @call-count)))))))++(deftest internal-metabot-scope-test+ (testing ":verified-only? is true only when the premium feature + use_verified_content both apply"+ (doseq [{:keys [features use-verified? expected-verified?]}+ [{:features #{} :use-verified? false :expected-verified? false}+ {:features #{} :use-verified? true :expected-verified? false}+ {:features #{:content-verification} :use-verified? false :expected-verified? false}+ {:features #{:content-verification} :use-verified? true :expected-verified? true}]]+ (testing (format "features=%s use_verified_content=%s" (pr-str features) use-verified?)+ (mt/with-premium-features features+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)+ {:use_verified_content use-verified? :collection_id nil}+ (is (= {:verified-only? expected-verified? :collection-id nil}+ (metabot-scope/internal-metabot-scope))))))))+ (testing ":collection-id is read straight from the internal Metabot row regardless of premium features"+ (mt/with-temp [:model/Collection {coll-id :id} {:name "metabot scope test coll"}]+ (mt/with-premium-features #{}+ (mt/with-temp-vals-in-db :model/Metabot (internal-metabot-id)+ {:use_verified_content false :collection_id coll-id}+ (is (= {:verified-only? false :collection-id coll-id}+ (metabot-scope/internal-metabot-scope))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljnew file mode 100644index 00000000000..54aa3d57a20--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -0,0 +1,1033 @@+(ns metabase-enterprise.data-complexity-score.complexity-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.data-complexity-score.complexity :as complexity]+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]+ ;; Load init for side-effect: exercises the transitive require of the task ns so+ ;; `scoring-task-registered-test` verifies init.clj's actual wiring path.+ [metabase-enterprise.data-complexity-score.init]+ [metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]+ [metabase-enterprise.data-complexity-score.settings :as data-complexity-score.settings]+ [metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]+ [metabase-enterprise.semantic-search.core :as semantic-search]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.embedders :as ss.embedders]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.analytics.core :as analytics]+ [metabase.analytics.snowplow-test :as snowplow-test]+ [metabase.audit-app.core :as audit]+ [metabase.collections.core :as collections]+ [metabase.collections.test-utils :as collections.tu]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private test-entity-ids (atom 0))++(defn- entity+ "Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`+ so every test entity is distinct (and so test failures print stable, readable ids)."+ [& {:keys [name kind field-count measure-names]+ :or {kind :table field-count 0 measure-names []}}]+ {:id (swap! test-entity-ids inc)+ :name name+ :kind kind+ :field-count field-count+ :measure-names measure-names})++(defn- mock-embedder+ "Build an embedder backed by a `{name -> vector-literal}` lookup table for tests."+ [name->vec-literal]+ (embedders/fn-embedder+ (fn [names]+ (mapv #(when-let [v (get name->vec-literal %)] (float-array v)) names))))++(deftest ^:parallel score-catalog-pure-test+ (testing "empty catalog scores zero"+ (is (=? {:total 0+ :components {:entity-count {:measurement 0.0 :score 0}+ :name-collisions {:measurement 0.0 :score 0}+ :synonym-pairs {:measurement 0.0 :score 0}+ :field-count {:measurement 0.0 :score 0}+ :repeated-measures {:measurement 0.0 :score 0}}}+ (#'complexity/score-catalog [] nil))))++ (testing "entity count contributes +10 per entity"+ (let [es [(entity :name "orders")+ (entity :name "customers")+ (entity :name "products")]]+ (is (=? {:total 30+ :components {:entity-count {:measurement 3.0 :score 30}}}+ (#'complexity/score-catalog es nil)))))++ (testing "name collisions stack linearly: 3 identical names = +200"+ (let [es [(entity :name "orders")+ (entity :name "orders")+ (entity :name "orders")]]+ (is (=? {:components {:entity-count {:measurement 3.0 :score 30}+ :name-collisions {:measurement 2.0 :score 200}}}+ (#'complexity/score-catalog es nil)))))++ (testing "collision detection is case-insensitive and trims whitespace"+ (let [es [(entity :name "Orders")+ (entity :name " orders ")+ (entity :name "ORDERS")]]+ (is (=? {:components {:name-collisions {:measurement 2.0 :score 200}}}+ (#'complexity/score-catalog es nil)))))++ (testing "field count contributes +1 per field, summed across entities"+ (let [es [(entity :name "a" :field-count 10)+ (entity :name "b" :field-count 25)]]+ (is (=? {:components {:field-count {:measurement 35.0 :score 35}}}+ (#'complexity/score-catalog es nil)))))++ (testing "repeated measures contribute +2 per repeat (measure name appearing on >1 entity)"+ (let [es [(entity :name "invoices" :measure-names ["revenue" "discount"])+ (entity :name "subscriptions" :measure-names ["revenue"])+ (entity :name "products" :measure-names ["price"])]]+ (is (=? {:components {:repeated-measures {:measurement 1.0 :score 2}}}+ (#'complexity/score-catalog es nil)))))++ (testing "nil embedder disables synonym scoring"+ (let [es [(entity :name "customers") (entity :name "clients")]]+ (is (=? {:components {:synonym-pairs {:measurement 0.0 :score 0}}}+ (#'complexity/score-catalog es nil))))))++(deftest ^:parallel synonym-scoring-test+ (testing "cosine similarity above threshold flags a synonym pair"+ (let [es [(entity :name "customers") (entity :name "clients")]+ embedder (mock-embedder {"customers" [1.0 0.0 0.0]+ "clients" [0.9 0.1 0.0]})]+ (is (=? {:components {:synonym-pairs {:measurement 1.0 :score 50}}}+ (#'complexity/score-catalog es embedder)))))++ (testing "orthogonal embeddings produce no synonym pairs"+ (let [es [(entity :name "customers") (entity :name "widgets")]+ embedder (mock-embedder {"customers" [1.0 0.0]+ "widgets" [0.0 1.0]})]+ (is (=? {:components {:synonym-pairs {:measurement 0.0 :score 0}}}+ (#'complexity/score-catalog es embedder)))))++ (testing "exact-name duplicates don't double-count as synonym pairs"+ (let [es [(entity :name "orders") (entity :name "orders") (entity :name "tickets")]+ embedder (mock-embedder {"orders" [1.0 0.0]+ "tickets" [0.0 1.0]})]+ (is (=? {:components {:name-collisions {:measurement 1.0 :score 100}+ :synonym-pairs {:measurement 0.0 :score 0}}}+ (#'complexity/score-catalog es embedder)))))++ (testing "entities without a vector from the embedder are simply skipped"+ (let [es [(entity :name "customers") (entity :name "clients") (entity :name "ghost")]+ ;; "ghost" is missing → not considered. The remaining two are synonyms.+ embedder (mock-embedder {"customers" [1.0 0.0]+ "clients" [0.99 0.01]})]+ (is (=? {:components {:synonym-pairs {:measurement 1.0 :score 50}}}+ (#'complexity/score-catalog es embedder)))))++ (testing "embedder failure cascades nil through the catalog (no zero-fallback)"+ (let [es [(entity :name "customers") (entity :name "clients")]+ embedder (fn [_] (throw (ex-info "boom" {})))]+ (is (=? {:total nil+ :components {:synonym-pairs {:measurement nil :score nil :error "boom"}+ ;; Sibling sub-scores still compute their real values — only the rollup+ ;; cascades nil — so consumers can still see the unaffected dimensions.+ :entity-count {:measurement 2.0 :score 20}}}+ (#'complexity/score-catalog es embedder)))))++ (testing "throwable with a nil/blank message still records :error as a nonblank string"+ ;; Regression: we must keep :error present so an embedder failure is distinguishable from a+ ;; genuine zero-synonym result. Fall back to the exception class name.+ (let [es [(entity :name "customers") (entity :name "clients")]+ synonym-of #(get-in (#'complexity/score-catalog es %) [:components :synonym-pairs])]+ (doseq [[label embedder expected] [["nil message" (fn [_] (throw (NullPointerException.)))+ "java.lang.NullPointerException"]+ ["blank message" (fn [_] (throw (RuntimeException. " ")))+ "java.lang.RuntimeException"]]]+ (testing label+ (let [sub (synonym-of embedder)]+ (is (nil? (:score sub)))+ (is (= expected (:error sub))+ (format ":error must be a nonblank string when the throwable's message is %s" label))))))))++(deftest ^:sequential complexity-scores-metabot-scope-opt-test+ (testing ":verified-only? true flows the caller's metabot-scope through to enumerate-catalogs"+ (let [captured-scope (atom nil)]+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (fn [scope]+ (reset! captured-scope scope)+ {:library []+ :universe [(entity :name "orders") (entity :name "widgets")]+ :metabot [(entity :name "orders")]})]+ (let [{:keys [universe metabot]} (complexity/complexity-scores+ :embedder nil+ :metabot-scope {:verified-only? true :collection-id nil})]+ (is (= {:verified-only? true :collection-id nil} @captured-scope)+ "enumerate-catalogs was invoked with the caller's scope")+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))+ (is (= 2.0 (get-in universe [:components :entity-count :measurement])))))))+ (testing ":collection-id alone also flows through to enumerate-catalogs (no verified flag required)"+ (let [captured-scope (atom nil)]+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (fn [scope]+ (reset! captured-scope scope)+ {:library []+ :universe [(entity :name "orders") (entity :name "widgets")]+ :metabot [(entity :name "orders")]})]+ (let [{:keys [metabot]} (complexity/complexity-scores+ :embedder nil+ :metabot-scope {:verified-only? false :collection-id 42})]+ (is (= {:verified-only? false :collection-id 42} @captured-scope))+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))))))+ (testing "empty scope (or no :metabot-scope opt) still runs enumerate-catalogs with that scope so metabot's table-visibility filter still narrows the catalog"+ ;; Regression: we used to reuse the :universe score verbatim when scope was empty. That hid the+ ;; fact that Metabot's table visibility (`:visibility_type nil`, non-routed DB) already narrows+ ;; the catalog even before Card scoping kicks in.+ (doseq [scope [nil {} {:verified-only? false :collection-id nil}]]+ (let [captured-scope (atom ::not-called)]+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (fn [s]+ (reset! captured-scope s)+ {:library []+ :universe [(entity :name "orders") (entity :name "widgets")]+ :metabot [(entity :name "orders")]})]+ (let [{:keys [universe metabot]} (complexity/complexity-scores+ :embedder nil+ :metabot-scope scope)]+ (is (= scope @captured-scope)+ (format "enumerate-catalogs was invoked with the caller's (possibly empty) scope=%s"+ (pr-str scope)))+ (is (= 1.0 (get-in metabot [:components :entity-count :measurement])))+ (is (= 2.0 (get-in universe [:components :entity-count :measurement])))))))))++(deftest ^:sequential metabot-catalog-excludes-hidden-tables-test+ (testing ":metabot tables filter out hidden (`visibility_type` non-nil) and routed-DB tables so the+ catalog matches what Metabot/search can actually surface"+ ;; Regression: `:metabot` used to count every active non-audit table, which overcounted on+ ;; instances with hidden tables or routed-database tables — Metabot never surfaces those.+ (mt/with-temp+ [:model/Database {router-db :id} {:name "Router DB"}+ :model/Database {routed-db :id} {:name "Routed DB" :router_database_id router-db}+ :model/Database {plain-db :id} {:name "Plain DB"}+ :model/Table {visible :id} {:db_id plain-db :name "visible_table"+ :active true :visibility_type nil}+ :model/Table {hidden :id} {:db_id plain-db :name "hidden_table"+ :active true :visibility_type "hidden"}+ :model/Table {technical :id} {:db_id plain-db :name "technical_table"+ :active true :visibility_type "technical"}+ :model/Table {routed :id} {:db_id routed-db :name "routed_table"+ :active true :visibility_type nil}]+ (let [metabot-entities (:metabot (#'complexity/enumerate-catalogs nil))+ ids (into #{} (comp (filter #(= :table (:kind %))) (map :id)) metabot-entities)]+ (testing "visible non-routed table is included"+ (is (contains? ids visible)))+ (testing "hidden and technical tables are excluded"+ (is (not (contains? ids hidden))+ "tables with visibility_type=hidden are filtered out")+ (is (not (contains? ids technical))+ "tables with visibility_type=technical are filtered out"))+ (testing "tables on a routed database are excluded"+ (is (not (contains? ids routed))+ "tables whose db has router_database_id are filtered out"))))))++(deftest ^:sequential metabot-collection-scope-ids-test+ (testing "nil collection-id returns nil (no Metabot collection scope configured)"+ (is (nil? (#'complexity/metabot-collection-scope-ids nil))))+ (testing "valid collection-id returns the id plus its descendants"+ (mt/with-temp [:model/Collection {parent :id} {:name "Scope parent" :location "/"}+ :model/Collection {child :id} {:name "Scope child" :location (format "/%d/" parent)}]+ (is (= #{parent child}+ (#'complexity/metabot-collection-scope-ids parent)))))+ (testing "invalid/deleted collection-id still returns a singleton set with the raw id"+ ;; The live `metabot-metrics-and-models-query` filters on the raw `collection_id` even when+ ;; the collection row is missing (stale Metabot scope value). If we dropped the filter here,+ ;; the :metabot catalog would overcount and drift back toward :universe.+ (let [ghost-id Integer/MAX_VALUE]+ (is (nil? (t2/select-one :model/Collection :id ghost-id))+ "pre-check: the phantom id isn't actually a real collection")+ (is (= #{ghost-id}+ (#'complexity/metabot-collection-scope-ids ghost-id))))))++(deftest ^:parallel fn-embedder-test+ (testing "normalizes names, dedupes, zips vectors by position, and omits entries with no vector"+ (let [known-vectors {"foo" (float-array [1.0 0.0])+ "bar" (float-array [0.0 1.0])+ "baz" (float-array [0.5 0.5])}+ embedder (embedders/fn-embedder (partial mapv known-vectors))+ result (embedder [{:name "Foo"}+ {:name " BAR"}+ {:name " Foo \t \n"}+ {:name "missing"}])]+ (is (=? {"foo" (known-vectors "foo")+ "bar" (known-vectors "bar")}+ result)))))++(deftest ^:sequential library-empty-when-no-library-collection-test+ (testing "on an instance with no Library collection, the library score is zero and universe still reports"+ (collections.tu/without-library+ (mt/with-temp [:model/Database {db-id :id} {:name "No-library Test DB"}+ :model/Table _ {:db_id db-id :name "contributes_to_universe" :active true}]+ (let [{:keys [library universe]} (complexity/complexity-scores :embedder nil)]+ (testing "library is empty (no collection tree)"+ (is (= {:total 0+ :components {:entity-count {:measurement 0.0 :score 0}+ :name-collisions {:measurement 0.0 :score 0}+ :synonym-pairs {:measurement 0.0 :score 0}+ :field-count {:measurement 0.0 :score 0}+ :repeated-measures {:measurement 0.0 :score 0}}}+ library)))+ (testing "universe still enumerates appdb content (our temp table + whatever else is there)"+ (is (pos? (:total universe)))))))))++(deftest ^:sequential library-excludes-audit-content-test+ (testing "published audit-db content in the Library tree is excluded so :library stays a subset of :universe"+ ;; Regression: library-entities previously didn't filter on audit/audit-db-id while universe/metabot did.+ ;; An audit card/table placed in the Library could push :library past :universe and break the new+ ;; subset invariant (the hermetic tests below assume library ⊆ universe on every component).+ (mt/with-temp [:model/Collection {lib-id :id} {:type collections/library-collection-type+ :name "Library"+ :location "/"}+ :model/Collection {data-id :id} {:type collections/library-data-collection-type+ :name "Data"+ :location (format "/%d/" lib-id)}+ :model/Collection {mets-id :id} {:type collections/library-metrics-collection-type+ :name "Metrics"+ :location (format "/%d/" lib-id)}+ :model/Database {audit-db :id} {:name "Fake Audit DB"}+ :model/Database {real-db :id} {:name "Non-audit DB"}+ ;; Audit-db tables published into the Library tree — MUST be excluded.+ :model/Table _ {:db_id audit-db :name "audit_events"+ :active true :is_published true+ :collection_id data-id}+ ;; Audit-db metric card in the Library — MUST be excluded.+ :model/Card _ {:database_id audit-db :type :metric :name "Audit Revenue"+ :archived false :collection_id mets-id}+ ;; Non-audit table in the Library — must still count.+ :model/Table _ {:db_id real-db :name "orders"+ :active true :is_published true+ :collection_id data-id}+ ;; Non-audit metric card in the Library — must still count.+ :model/Card _ {:database_id real-db :type :metric :name "Real Revenue"+ :archived false :collection_id mets-id}]+ (with-redefs [audit/audit-db-id audit-db]+ (let [{:keys [library]} (complexity/complexity-scores :embedder nil)]+ (is (= 2.0 (get-in library [:components :entity-count :measurement]))+ "only the two non-audit entities count (audit table + audit metric card excluded)"))))))++;; We're only reading the method table via `methods`, not calling the impure `!` fn — safe in parallel.+#_{:clj-kondo/ignore [:metabase/validate-deftest]}+(deftest ^:parallel scoring-task-registered-test+ (testing "loading the data-complexity-score init namespace registers the scoring task's `task/init!` method"+ (is (contains? (methods task/init!)+ :metabase-enterprise.data-complexity-score.task.complexity-score/DataComplexityScoring))))++(deftest ^:parallel search-index-embedder-degrades-gracefully-test+ (testing "returns {} when semantic-search index isn't available (no throw)"+ (is (= {} (semantic-search/search-index-embedder+ [(entity :name "orders" :kind :table)])))))++(deftest ^:sequential complexity-scores-with-real-search-index-embedder-test+ (testing "complexity-scores drives the real search-index-embedder + pgvector end-to-end"+ ;; Self-gated on MB_PGVECTOR_DB_URL — CI without semantic-search infra skips this;+ ;; locally with pgvector running it exercises the pgvector read path that all other+ ;; embedder tests in this ns stub out (via mt/with-dynamic-fn-redefs on try-active-index-state /+ ;; fetch-batch). Uses `:mock-indexed` so the *embedding model* is a lookup-table — but+ ;; the resulting vectors are real rows in the index table that search-index-embedder+ ;; queries via SQL.+ (when semantic.db.datasource/db-url+ (let [captured (atom nil)]+ (mt/with-premium-features #{:semantic-search}+ (mt/as-admin+ (semantic.tu/with-test-db! {:mode :mock-indexed}+ (reset! captured (complexity/complexity-scores+ :embedder semantic-search/search-index-embedder)))))+ (is (=? {:meta {:embedding-model {:provider "mock" :model-name "model"}}+ :universe {:components {:synonym-pairs {:measurement number?+ :score nat-int?}}}}+ @captured)+ "embedder returned vectors from pgvector and the synonym axis produced a real measurement")))))++(deftest ^:sequential search-index-embedder-propagates-read-failures-test+ (testing "pgvector read failures propagate so the caller can flag a degraded synonym axis"+ ;; Regression guard: before this, the embedder swallowed read exceptions and returned {}, which+ ;; looked indistinguishable from \"no synonym matches.\" A transient search-index failure+ ;; silently underreported complexity with no `:error` on the Snowplow payload.+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock :table-name "t" :model nil})+ ss.embedders/fetch-batch+ (fn [& _] (throw (ex-info "pgvector read failed" {})))]+ (testing "the embedder itself throws rather than swallowing the failure"+ (is (thrown-with-msg? Throwable #"pgvector read failed"+ (semantic-search/search-index-embedder+ [{:id 1 :name "orders" :kind :table}]))))+ (testing "score-synonym-pairs converts the propagated failure into :error on the sub-score"+ (let [es [(entity :name "customers") (entity :name "clients")]]+ (is (=? {:components {:synonym-pairs {:measurement nil+ :score nil+ :error string?}}}+ (#'complexity/score-catalog es semantic-search/search-index-embedder))))))))++(defn- stub-fetch-batch+ "Build a `fetch-batch` stub backed by a map of expected pair-sets -> rows. Each+ invocation looks up rows by the incoming `(model, model_id)` pairs, so the stub+ validates the real batching contract (which pair-sets are requested) without+ coupling to call order. Unknown or duplicate pair-sets throw. Returns+ `{:unseen <atom>, :stub <fn>}`; after the call under test, assert+ `(empty? @unseen)` to catch missing invocations."+ [expected]+ (let [unseen (atom (set (keys expected)))]+ {:unseen unseen+ :stub (fn [_ _ pairs]+ (let [k (mapv vec pairs)]+ (if (contains? @unseen k)+ (do (swap! unseen disj k)+ (get expected k))+ (throw (ex-info (format "fetch-batch called with unexpected or duplicate pairs: %s (remaining expected: %s)"+ (pr-str k) (pr-str @unseen))+ {:pairs k :remaining @unseen})))))}))++(deftest ^:sequential search-index-embedder-global-dedup-test+ (testing "global dedup picks lowest numeric model_id regardless of batch boundaries"+ ;; Stub at fetch-batch so the real batching path executes. With batch-size 1, each entity+ ;; pair lands in its own SQL batch, so the two duplicates come back from separate partitions+ ;; and must be merged by the fold. The stub keys expected batches by `(model, model_id)`+ ;; pair-seq, so a regression in `entity-type->search-model` or any missing batch fails here.+ (let [winner-vec (float-array [1.0 0.0 0.0])+ loser-vec (float-array [0.0 1.0 0.0])+ {:keys [unseen stub]}+ (stub-fetch-batch+ {;; table with model_id "10" (loser — higher id)+ [["table" "10"]] [{:name "Orders" :model_id "10" :model "table" :embedding loser-vec}]+ ;; table with model_id "2" (winner — lower id)+ [["table" "2"]] [{:name "orders" :model_id "2" :model "table" :embedding winner-vec}]})]+ (with-redefs [ss.embedders/fetch-batch-size 1]+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock :table-name "t" :model nil})+ ss.embedders/fetch-batch stub]+ (let [result (semantic-search/search-index-embedder+ [{:id 10 :name "Orders" :kind :table}+ {:id 2 :name "orders" :kind :table}])]+ (is (= 1 (count result)) "duplicate normalized names collapse to one entry")+ (is (= (seq winner-vec) (seq (get result "orders")))+ "the row with the lowest numeric model_id wins")+ (is (empty? @unseen)+ "every expected fetch-batch pair-set must be requested"))))))++ (testing "cross-model duplicates: lowest model_id wins, model is secondary tie-break"+ ;; :kind :question maps to "card" and :kind :table maps to "table" via entity-type->search-model,+ ;; so the real (model, model_id) query contract is exercised. Both have id 5, so model_ids tie+ ;; and the lexicographically smaller model ("card" < "table") must win. The stub is keyed by+ ;; pairs so a regression in entity-type->search-model (e.g. :question no longer mapping to+ ;; "card") throws from the stub instead of passing silently.+ (let [card-vec (float-array [1.0 0.0])+ table-vec (float-array [0.0 1.0])+ {:keys [unseen stub]}+ (stub-fetch-batch+ {;; card (from :question entity)+ [["card" "5"]] [{:name "Revenue" :model_id "5" :model "card" :embedding card-vec}]+ ;; table (from :table entity)+ [["table" "5"]] [{:name "revenue" :model_id "5" :model "table" :embedding table-vec}]})]+ (with-redefs [ss.embedders/fetch-batch-size 1]+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock :table-name "t" :model nil})+ ss.embedders/fetch-batch stub]+ (let [result (semantic-search/search-index-embedder+ [{:id 5 :name "Revenue" :kind :question}+ {:id 5 :name "revenue" :kind :table}])]+ (is (= 1 (count result)))+ (is (= (seq card-vec) (seq (get result "revenue")))+ "when model_ids tie, the lexicographically smaller model wins (card < table)")+ (is (empty? @unseen)+ "every expected fetch-batch pair-set must be requested")))))))++(deftest ^:sequential search-index-embedder-cross-batch-dedup-test+ (testing "duplicates split across separate fetch batches are resolved globally"+ ;; Mock at fetch-batch so the real partition-all fold executes. With batch-size 1, each entity+ ;; pair lands in its own SQL batch. The stub is keyed by the expected `(model, model_id)`+ ;; pair-seq, so it validates the batching contract — `entity-type->search-model` mapping+ ;; (`:question` → "card", `:table` → "table") is exercised, not assumed.+ (let [winner-vec (float-array [1.0 0.0 0.0])+ loser-vec (float-array [0.0 1.0 0.0])+ other-vec (float-array [0.0 0.0 1.0])+ {:keys [unseen stub]}+ (stub-fetch-batch+ {;; table with model_id "7"+ [["table" "7"]] [{:name "Orders" :model_id "7" :model "table" :embedding loser-vec}]+ ;; card with model_id "3" — should win globally+ [["card" "3"]] [{:name "orders" :model_id "3" :model "card" :embedding winner-vec}]+ ;; different entity, no collision+ [["table" "1"]] [{:name "Products" :model_id "1" :model "table" :embedding other-vec}]})]+ (with-redefs [ss.embedders/fetch-batch-size 1]+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock :table-name "t" :model nil})+ ss.embedders/fetch-batch stub]+ (let [result (semantic-search/search-index-embedder+ [{:id 7 :name "Orders" :kind :table}+ {:id 3 :name "orders" :kind :question}+ {:id 1 :name "Products" :kind :table}])]+ (is (= 2 (count result)) "two distinct normalized names survive dedup")+ (is (= (seq winner-vec) (seq (get result "orders")))+ "lowest model_id wins across batch boundaries")+ (is (some? (get result "products"))+ "non-colliding entity is retained")+ (is (empty? @unseen)+ "every expected fetch-batch pair-set must be requested"))))))++ (testing "cross-batch, cross-model duplicates: model_id primary, model secondary"+ ;; Same normalized name from three different batches and three different model types.+ ;; model_id "5" appears twice (card + dataset); model_id "12" is in a third batch.+ ;; Expected winner: model_id "5", model "card" (lowest id, then lexicographic: "card" < "dataset").+ ;; Entity kinds cover :table → "table", :model → "dataset", :question → "card".+ (let [winner-vec (float-array [1.0 0.0])+ same-id-other-model (float-array [0.0 1.0])+ higher-id-vec (float-array [0.5 0.5])+ {:keys [unseen stub]}+ (stub-fetch-batch+ {;; high model_id, model "table"+ [["table" "12"]] [{:name "Revenue" :model_id "12" :model "table" :embedding higher-id-vec}]+ ;; low model_id, model "dataset"+ [["dataset" "5"]] [{:name "revenue" :model_id "5" :model "dataset" :embedding same-id-other-model}]+ ;; low model_id, model "card" — card < dataset so this wins+ [["card" "5"]] [{:name "REVENUE" :model_id "5" :model "card" :embedding winner-vec}]})]+ (with-redefs [ss.embedders/fetch-batch-size 1]+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock :table-name "t" :model nil})+ ss.embedders/fetch-batch stub]+ (let [result (semantic-search/search-index-embedder+ [{:id 12 :name "Revenue" :kind :table}+ {:id 5 :name "revenue" :kind :model}+ {:id 5 :name "REVENUE" :kind :question}])]+ (is (= 1 (count result)) "all three collapse to one normalized name")+ (is (= (seq winner-vec) (seq (get result "revenue")))+ "model_id 5 + model card wins (lowest id, then lexicographic: card < dataset)")+ (is (empty? @unseen)+ "every expected fetch-batch pair-set must be requested")))))))++(deftest ^:parallel prefer-new-row-test+ (let [prefer? #'ss.embedders/prefer-new-row?]+ (are [expected? new-row prior-row]+ (expected? (prefer? new-row prior-row))+ ;; lower numeric model_id wins+ true? {:mid 2 :model_id "2" :model "card"} {:mid 10 :model_id "10" :model "card"}+ false? {:mid 10 :model_id "10" :model "card"} {:mid 2 :model_id "2" :model "card"}+ ;; equal model_ids tie-break on model name+ true? {:mid 5 :model_id "5" :model "card"} {:mid 5 :model_id "5" :model "table"}+ false? {:mid 5 :model_id "5" :model "table"} {:mid 5 :model_id "5" :model "card"}+ ;; same parsed number but different raw strings → tie-break on raw model_id before model+ true? {:mid 2 :model_id "02" :model "card"} {:mid 2 :model_id "2" :model "card"}+ false? {:mid 2 :model_id "2" :model "card"} {:mid 2 :model_id "02" :model "card"}+ ;; numeric always beats non-numeric+ true? {:mid 99 :model_id "99" :model "card"} {:mid nil :model_id "abc" :model "card"}+ false? {:mid nil :model_id "abc" :model "card"} {:mid 1 :model_id "1" :model "card"}+ ;; both non-numeric: lexicographic on model_id string+ true? {:mid nil :model_id "abc" :model "card"} {:mid nil :model_id "xyz" :model "card"}+ false? {:mid nil :model_id "xyz" :model "card"} {:mid nil :model_id "abc" :model "card"})))++(deftest ^:sequential meta-embedding-model-absent-when-unavailable-test+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [] :universe [] :metabot []})]+ (testing ":embedding-model key is absent from :meta when the search index is unreachable"+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state (constantly nil)]+ ;; Pass the real search-index-embedder so the identity check in complexity-scores succeeds,+ ;; but the embedder returns {} because try-active-index-state is nil.+ (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]+ (is (not (contains? meta :embedding-model))))))+ (testing ":embedding-model key is present in :meta when the active model is non-nil"+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock+ :table-name "mock_table"+ :model {:provider "openai"+ :model-name "text-embedding-3-small"}})+ ss.embedders/fetch-batch (constantly [])]+ (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]+ (is (= {:provider "openai" :model-name "text-embedding-3-small"}+ (:embedding-model meta))))))))++(deftest ^:sequential active-embedding-model-reads-from-active-index-test+ (testing "active-embedding-model returns the model from the active index, not the configured setting"+ (let [active-model {:provider "openai" :model-name "text-embedding-ada-002"}]+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock+ :table-name "mock_table"+ :model active-model})]+ (is (= {:provider "openai" :model-name "text-embedding-ada-002"}+ (semantic-search/active-embedding-model))))))+ (testing "active-embedding-model returns nil when the index state has no model"+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock+ :table-name "mock_table"+ :model nil})]+ (is (nil? (semantic-search/active-embedding-model)))))+ (testing "active-embedding-model returns nil when the index is unreachable"+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state (constantly nil)]+ (is (nil? (semantic-search/active-embedding-model))))))++(defn- complexity-events!+ "Drain the fake Snowplow collector and return only complexity events."+ []+ (->> (snowplow-test/pop-event-data-and-user-id!)+ (map :data)+ (filter #(= "data_complexity_scoring" (get % "event")))))++(def ^:private leaf-key->group+ "Mirrors `complexity/component->group` for test assertions. Kept in snake-case string form+ (matching how keys land in the Snowplow payload)."+ {"entity_count" "size"+ "field_count" "size"+ "name_collisions" "ambiguity"+ "synonym_pairs" "ambiguity"+ "repeated_measures" "ambiguity"})++(defn- snake [k] (-> k name (str/replace "-" "_")))++(defn- expected-keys-for-catalog+ "For a catalog result, return `#{[key score], ...}` matching what emit-snowplow! should emit:+ grand `total`, one `<group>.total` per group, and one `<group>.<leaf>` per sub-component."+ [{:keys [total components]}]+ (let [leaves (into {} (map (fn [[k sub]] [(snake k) (:score sub)]) components))+ group-total (reduce-kv (fn [acc leaf score]+ (update acc (leaf-key->group leaf) (fnil + 0) score))+ {} leaves)]+ (set (concat [["total" total]]+ (for [[g s] group-total] [(str g ".total") s])+ (for [[leaf s] leaves] [(str (leaf-key->group leaf) "." leaf) s])))))++(deftest ^:sequential emit-snowplow-publishes-total-and-each-subscore-test+ (testing "one event per (catalog × key) — grand total, group rollups, and leaves — with correct scores"+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")+ (entity :name "customers")]+ :universe [(entity :name "orders")+ (entity :name "customers")+ (entity :name "widgets")]+ :metabot []})]+ ;; Drain any startup/setting events so we only assert on emissions from the call below.+ (snowplow-test/pop-event-data-and-user-id!)+ (let [{:keys [library universe metabot]} (complexity/complexity-scores :embedder nil)+ events (complexity-events!)+ expected (into #{}+ (for [[catalog result] {"library" library+ "universe" universe+ "metabot" metabot}+ [k score] (expected-keys-for-catalog result)]+ [catalog k score]))+ actual (into #{}+ (map (fn [e] [(get e "catalog") (get e "key") (get e "score")]) events))]+ (is (= (count expected) (count events))+ "every (catalog, key) is emitted exactly once — no duplicates")+ (is (= expected actual)+ "every (catalog, key) pair carries the matching score from the result")+ (testing "every event carries the event name, formula version, and parameters map"+ (is (every? (fn [e]+ (and (= "data_complexity_scoring" (get e "event"))+ (integer? (get e "formula_version"))+ (map? (get e "parameters"))+ (number? (get-in e ["parameters" "synonym_threshold"]))))+ events))))))))++(deftest ^:sequential emit-snowplow-includes-measurement-for-sub-components-test+ (testing "each leaf event carries the raw pre-score measurement; totals and group-totals do not"+ (snowplow-test/with-fake-snowplow-collector+ ;; Library: 3 entities, one collision pair, 5 fields total.+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders" :field-count 2)+ (entity :name "orders" :field-count 0)+ (entity :name "widgets" :field-count 3)]+ :universe []+ :metabot []})]+ (snowplow-test/pop-event-data-and-user-id!)+ (complexity/complexity-scores :embedder nil)+ (let [by-key (->> (complexity-events!)+ (filter #(= "library" (get % "catalog")))+ (into {} (map (juxt #(get % "key") identity))))]+ (testing "aggregate totals (grand and per-group) have no measurement key"+ (is (not (contains? (get by-key "total") "measurement")))+ (is (not (contains? (get by-key "size.total") "measurement")))+ (is (not (contains? (get by-key "ambiguity.total") "measurement"))))+ (testing "size leaves carry raw counts"+ (is (= 3.0 (get-in by-key ["size.entity_count" "measurement"])))+ (is (= 5.0 (get-in by-key ["size.field_count" "measurement"]))))+ (testing "ambiguity leaves carry their respective measurements"+ (is (= 1.0 (get-in by-key ["ambiguity.name_collisions" "measurement"])))+ (is (= 0.0 (get-in by-key ["ambiguity.synonym_pairs" "measurement"])))+ (is (= 0.0 (get-in by-key ["ambiguity.repeated_measures" "measurement"])))))))))++(deftest ^:sequential emit-snowplow-cascades-nil-on-embedder-failure-test+ (testing "embedder failure cascades nil through aggregates without skipping any events"+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "customers")+ (entity :name "clients")]+ :universe []+ :metabot []})]+ (snowplow-test/pop-event-data-and-user-id!)+ (complexity/complexity-scores :embedder (fn [_] (throw (ex-info "embedder boom" {}))))+ (let [by-key (->> (complexity-events!)+ (filter #(= "library" (get % "catalog")))+ (into {} (map (juxt #(get % "key") identity))))]+ (testing ":error from the synonym-pair scorer reaches the Snowplow leaf event"+ (is (= "embedder boom" (get-in by-key ["ambiguity.synonym_pairs" "error"]))))+ (testing ":error is only present on the originating leaf — aggregates use null score instead"+ (is (not-any? #(contains? % "error")+ (vals (dissoc by-key "ambiguity.synonym_pairs")))))+ (testing "the failed leaf publishes null :score (no zero-fallback)"+ (is (nil? (get-in by-key ["ambiguity.synonym_pairs" "score"]))))+ (testing "aggregates that include the failed leaf cascade null :score"+ (is (nil? (get-in by-key ["ambiguity.total" "score"])))+ (is (nil? (get-in by-key ["total" "score"]))))+ (testing "unaffected aggregates keep their numeric :score (cascade is leaf-scoped, not catalog-wide)"+ ;; size group has no synonym dependency, so its rollup must still be a real number even+ ;; when ambiguity falls through. Catches a regression where the cascade goes too far.+ (is (number? (get-in by-key ["size.total" "score"])))))))))++(deftest ^:sequential emit-snowplow-truncates-error-to-schema-max-test+ (testing "a pathologically long exception message is truncated so it doesn't fail schema validation"+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "customers")+ (entity :name "clients")]+ :universe []+ :metabot []})]+ (snowplow-test/pop-event-data-and-user-id!)+ (let [huge (apply str (repeat 5000 "x"))]+ (complexity/complexity-scores :embedder (fn [_] (throw (ex-info huge {}))))+ (let [err (->> (complexity-events!)+ (filter #(= "ambiguity.synonym_pairs" (get % "key")))+ first+ (#(get % "error")))]+ (is (= 1024 (count err))+ "error is clipped to the schema's maxLength of 1024")))))))++(deftest ^:sequential emit-snowplow-includes-embedding-model-meta-test+ (testing "every event's parameters carry embedding_model_provider/name when the search-index embedder is active"+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state+ (constantly {:pgvector :mock+ :table-name "mock_table"+ :model {:provider "openai"+ :model-name "text-embedding-3-small"}})+ ss.embedders/fetch-batch (constantly [])+ complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")]+ :universe [(entity :name "orders")]+ :metabot []})]+ (snowplow-test/pop-event-data-and-user-id!)+ (complexity/complexity-scores :embedder semantic-search/search-index-embedder)+ (let [events (complexity-events!)]+ (is (seq events) "sanity: events were emitted")+ (is (every? #(= "openai" (get-in % ["parameters" "embedding_model_provider"])) events))+ (is (every? #(= "text-embedding-3-small" (get-in % ["parameters" "embedding_model_name"])) events)))))))++(deftest ^:sequential emit-snowplow-failure-is-swallowed-test+ (testing "emission failure is caught; complexity-scores still returns the score and logs a warning"+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")]+ :universe [(entity :name "orders")]+ :metabot []})+ analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :warn]]+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (=? {:library {:total 10 :components {:entity-count {:measurement 1.0 :score 10}}}+ :universe {:total 10 :components {:entity-count {:measurement 1.0 :score 10}}}}+ result))+ (is (some #(re-find #"Failed to publish complexity score" (:message %))+ (messages))+ "a warning about the publish failure was logged"))))))++(deftest ^:sequential local-info-log-is-emitted-even-when-snowplow-fails-test+ (testing "the 'Semantic complexity score' info log fires independently of Snowplow emission"+ ;; Guards two regressions together: local logging being removed, and local logging being+ ;; gated on successful telemetry (so a broken collector would silence the operator-visible log).+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")]+ :universe [(entity :name "orders")]+ :metabot []})+ analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]+ (complexity/complexity-scores :embedder nil)+ (is (some #(and (= :info (:level %))+ (re-find #"Semantic complexity score" (:message %)))+ (messages))+ "the score was logged locally at :info even though Snowplow emission threw")))))++(deftest ^:sequential scheduled-task-logs-score-test+ (testing "the scheduled task body runs complexity-scores so operators see a score line in the logs"+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [(entity :name "orders")]+ :universe [(entity :name "orders")]+ :metabot []})]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true]+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-complexity-score.complexity :info]]+ (#'task.complexity-score/run-scoring! "test-fp")+ (is (some #(and (= :info (:level %))+ (re-find #"Semantic complexity score" (:message %)))+ (messages))+ "the scheduled task produced the expected info log"))))))++(deftest ^:sequential complexity-score-library-hermetic-test+ (testing "library score is computed over exactly the Library collection tree — known inputs produce known scores"+ ;; The library tree gets a fixed set of tables, fields, measures, and metric cards. One extra+ ;; collection + table + card sit outside the library so the universe is a strict superset.+ (mt/with-temp+ [:model/Collection {lib-id :id} {:type collections/library-collection-type+ :name "Library"+ :location "/"}+ :model/Collection {data-id :id} {:type collections/library-data-collection-type+ :name "Data"+ :location (format "/%d/" lib-id)}+ :model/Collection {metrics-id :id} {:type collections/library-metrics-collection-type+ :name "Metrics"+ :location (format "/%d/" lib-id)}+ :model/Collection {other-id :id} {:name "Outside" :location "/"}+ :model/Database {db-id :id} {:name "Hermetic Library DB"}+ ;; Library tables. clients + customers are a synonym pair for the mock embedder below.+ :model/Table {t1 :id} {:db_id db-id :name "orders"+ :active true :is_published true :collection_id data-id}+ :model/Table {t2 :id} {:db_id db-id :name "subscriptions"+ :active true :is_published true :collection_id data-id}+ :model/Table _ {:db_id db-id :name "clients"+ :active true :is_published true :collection_id data-id}+ :model/Table _ {:db_id db-id :name "customers"+ :active true :is_published true :collection_id data-id}+ ;; Non-library tables: active (→ in universe) but not published into the library tree.+ ;; audit_events + audit_log are another synonym pair via the mock embedder.+ :model/Table {t-audit :id} {:db_id db-id :name "audit_events"+ :active true :is_published false :collection_id other-id}+ :model/Table _ {:db_id db-id :name "audit_log"+ :active true :is_published false :collection_id other-id}+ ;; Library fields: 2 on orders, 1 on subscriptions.+ :model/Field _ {:table_id t1 :name "id" :active true :base_type :type/Integer}+ :model/Field _ {:table_id t1 :name "total" :active true :base_type :type/Float}+ :model/Field _ {:table_id t2 :name "id" :active true :base_type :type/Integer}+ ;; Non-library fields on audit_events → bumps universe field-count above library.+ :model/Field _ {:table_id t-audit :name "audit_id"+ :active true :base_type :type/Integer}+ :model/Field _ {:table_id t-audit :name "audit_ts"+ :active true :base_type :type/DateTime}+ ;; Library measures: "revenue" on two library tables → one repeated-measure pair in library.+ :model/Measure _ {:table_id t1 :name "revenue"+ :creator_id (mt/user->id :rasta)+ :definition {} :archived false}+ :model/Measure _ {:table_id t2 :name "revenue"+ :creator_id (mt/user->id :rasta)+ :definition {} :archived false}+ ;; Non-library measure with the same name → bumps universe repeated-measures above library.+ :model/Measure _ {:table_id t-audit :name "revenue"+ :creator_id (mt/user->id :rasta)+ :definition {} :archived false}+ ;; Two library metric cards named "Revenue" → one name-collision pair in library.+ :model/Card _ {:database_id db-id :type :metric :name "Revenue"+ :archived false :collection_id metrics-id}+ :model/Card _ {:database_id db-id :type :metric :name "Revenue"+ :archived false :collection_id metrics-id}+ ;; Non-library card also named "Revenue" → 3 "Revenue" entities in universe → 2 collision pairs.+ :model/Card _ {:database_id db-id :type :model :name "Revenue"+ :archived false :collection_id other-id}]+ ;; Each name gets its own orthogonal dimension so only the intended pairs cluster.+ (let [embedder (mock-embedder {"orders" [1.0 0.0 0.0 0.0 0.0 0.0 0.0]+ "subscriptions" [0.0 1.0 0.0 0.0 0.0 0.0 0.0]+ "clients" [0.0 0.0 1.0 0.0 0.0 0.0 0.0]+ "customers" [0.0 0.0 0.99 0.1 0.0 0.0 0.0] ; ≈ clients (library)+ "revenue" [0.0 0.0 0.0 0.0 1.0 0.0 0.0] ; "Revenue" cards normalize here+ "audit_events" [0.0 0.0 0.0 0.0 0.0 1.0 0.0]+ "audit_log" [0.0 0.0 0.0 0.0 0.0 0.99 0.1]}) ; ≈ audit_events (universe-only)+ {:keys [library universe]} (complexity/complexity-scores :embedder embedder)]+ (testing "library reflects exactly what we put in the Library collection tree"+ ;; Library: 4 tables + 2 metric cards = 6 entities.+ ;; entity-count 6 × 10 = 60+ ;; name-collisions "revenue" (2 metric cards) = 1 pair × 100 = 100+ ;; synonym-pairs clients ↔ customers = 1 × 50 = 50+ ;; field-count orders(2) + subscriptions(1) + others(0) = 3 × 1 = 3+ ;; repeated-measures "revenue" on orders + subscriptions = 1 × 2 = 2+ ;; total 60 + 100 + 50 + 3 + 2 = 215+ (is (= {:total 215+ :components {:entity-count {:measurement 6.0 :score 60}+ :name-collisions {:measurement 1.0 :score 100}+ :synonym-pairs {:measurement 1.0 :score 50}+ :field-count {:measurement 3.0 :score 3}+ :repeated-measures {:measurement 1.0 :score 2}}}+ library)))+ (testing "universe is a strict superset of library on every component: every measurement and score is higher"+ ;; Note: :synonym-pairs is monotonic on this fixture but not in general — score-synonym-pairs+ ;; dedupes by normalized name and picks one embedding per name, so a universe-only entity+ ;; sharing a normalized name with a library entity could in theory flip which vector wins and+ ;; decrease the pair count/score. Our fixture doesn't hit that case; if this assertion ever+ ;; flakes, that's the reason.+ (doseq [component [:entity-count :name-collisions :synonym-pairs :field-count :repeated-measures]+ k [:measurement :score]+ :let [lib-v (get-in library [:components component k])+ uni-v (get-in universe [:components component k])]]+ (is (> uni-v lib-v)+ (format "universe %s %s (%s) should be strictly > library %s %s (%s)"+ component k uni-v component k lib-v))))+ (testing "universe total is strictly higher than library total"+ (is (> (:total universe) (:total library))))))))++(defn- stub-result+ "Build a `complexity/complexity-scores` stand-in whose metadata records whether publishing worked."+ [published?]+ (with-meta+ {:library {:total 0 :components {}} :universe {:total 0 :components {}}+ :metabot {:total 0 :components {}} :meta {}}+ {:metabase-enterprise.data-complexity-score.complexity/snowplow-published? published?}))++(deftest ^:sequential run-scoring-persists-fingerprint-only-on-successful-publish-test+ (testing "fingerprint advances only when Snowplow accepted the event — failed publish must leave+ the stale fingerprint in place so the next boot / cron retries"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ (testing "successful publish → fingerprint advances to the claim's fingerprint"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]+ (#'task.complexity-score/run-scoring! "fresh-fp")+ (is (= "fresh-fp" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint stamped from the claim fingerprint, not re-sampled at commit time"))))+ (testing "failed publish → fingerprint stays at the stale value for the next retry"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]+ (#'task.complexity-score/run-scoring! "fresh-fp")+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint preserved — next boot / cron will retry the emission")))))))++(deftest ^:sequential maybe-emit-boot-score-only-advances-fingerprint-on-successful-publish-test+ (testing "boot-time emission never advances the last-successful fingerprint on failure — the+ success fingerprint is separate from the in-progress scoring claim, so a scoring/publish+ failure (or a crash mid-run) leaves the fingerprint stale and the next boot retries"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ (testing "publish failure → fingerprint stays at the stale value (and claim is cleared)"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint unchanged — next boot/cron will retry the emission")+ (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))+ "scoring claim released so other paths can proceed without waiting for TTL"))))+ (testing "publish success → fingerprint advances to the new value (and claim is cleared)"+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint advanced to reflect the confirmed publish")+ (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))+ "scoring claim released after successful run")))))))++(deftest ^:sequential maybe-emit-boot-score-skips-when-another-path-holds-active-claim-test+ (testing "a fresh scoring claim (from a sibling node OR a concurrent cron tick) blocks duplicate+ emission on this node, even when the last-successful fingerprint is stale — prevents+ the boot and cron paths from both computing and publishing for the same fingerprint"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ ;; Serialize a sibling/cron claim for the same fingerprint, timestamped just now so+ ;; it's well inside the TTL.+ active-claim (pr-str {:fingerprint current-fp :claimed-at (#'task.complexity-score/now-ms)})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim active-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (false? @scoring-ran?)+ "scoring skipped because another path already claimed the current fingerprint")+ (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint untouched when the claim is skipped")+ (is (= active-claim (data-complexity-score.settings/data-complexity-scoring-claim))+ "other path's claim is preserved (we never took it, so we don't clear it)")))))))++(deftest ^:sequential maybe-emit-boot-score-reclaims-when-prior-claim-has-expired-test+ (testing "an expired (TTL-exceeded) scoring claim must not permanently suppress scoring — a+ crashed or orphaned claim from a prior run should be replaced and scoring proceed"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ ;; 1 hour ago — older than the 30-minute TTL, so this claim is treated as orphaned.+ expired-claim (pr-str {:fingerprint current-fp+ :claimed-at (- (#'task.complexity-score/now-ms)+ (* 60 60 1000))})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim expired-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (true? @scoring-ran?)+ "scoring ran because the prior claim had aged past the TTL")+ (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))+ "fingerprint advanced on successful publish after re-claim")))))))++(deftest ^:sequential maybe-emit-boot-score-does-not-clear-sibling-claim-after-ttl-takeover-test+ (testing "if our run outlives the TTL and another path legitimately re-claims, the original+ claimant's release-scoring-claim! must NOT wipe the replacement claim — doing so would+ reopen the duplicate-publish race (a third node/path would see no claim and run again)"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ ;; Stub scoring so that while this node is mid-run the persisted claim is replaced with a+ ;; sibling's fresh claim (different :owner). This simulates: our run hung past the 30-min+ ;; TTL → sibling saw an expired claim → sibling wrote its own → our run finally returns.+ (let [sibling-claim (pr-str {:fingerprint (#'task.complexity-score/current-fingerprint)+ :claimed-at (System/currentTimeMillis)+ :owner "sibling-node-owner-token"})]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim ""]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _]+ (data-complexity-score.settings/data-complexity-scoring-claim! sibling-claim)+ (stub-result true))]+ (task.complexity-score/maybe-emit-boot-score!)+ (is (= sibling-claim (data-complexity-score.settings/data-complexity-scoring-claim))+ "replacement claim preserved — our release was a compare-and-clear and the owners didn't match")))))))++(deftest ^:sequential cron-skips-when-boot-run-holds-active-claim-test+ (testing "the cron job's scoring call skips when the boot-time run is already mid-flight for the+ current fingerprint — the shared claim protocol serializes the two paths so they can't+ both compute and publish in parallel"+ (mt/with-dynamic-fn-redefs [metabot-scope/internal-metabot-scope (constantly {})]+ (let [current-fp (#'task.complexity-score/current-fingerprint)+ boot-claim (pr-str {:fingerprint current-fp+ :claimed-at (#'task.complexity-score/now-ms)+ :owner "boot-path-owner-token"})+ scoring-ran? (atom false)]+ (mt/with-temporary-setting-values [data-complexity-scoring-enabled true+ data-complexity-scoring-last-fingerprint "stale"+ data-complexity-scoring-claim boot-claim]+ (mt/with-dynamic-fn-redefs [complexity/complexity-scores+ (fn [& _] (reset! scoring-ran? true) (stub-result true))]+ ;; Exercise the cron's code path via the shared helper — this is exactly what the+ ;; Quartz job body calls. Re-sampling current-fingerprint does not matter because the+ ;; live value matches the one the boot path claimed.+ (#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-scoring!)+ (is (false? @scoring-ran?)+ "cron tick skipped because the boot run holds the scoring claim")+ (is (= boot-claim (data-complexity-score.settings/data-complexity-scoring-claim))+ "boot's claim preserved — cron never took it, so it doesn't clear it")))))))++(deftest ^:sequential complexity-scores-tags-publish-success-on-result-test+ (testing "complexity-scores stamps publish success/failure via metadata for schedule/boot callers"+ (mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs+ (constantly {:library [] :universe [] :metabot []})]+ (testing "successful publish → ::snowplow-published? true"+ (snowplow-test/with-fake-snowplow-collector+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (true? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result)))))))+ (testing "publish throw → ::snowplow-published? false (and the result is still returned)"+ (mt/with-dynamic-fn-redefs [analytics/track-event! (fn [& _] (throw (RuntimeException. "snowplow down")))]+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result)))))))+ (testing "snowplow disabled → ::snowplow-published? false so the fingerprint stays unadvanced"+ ;; `track-event!` no-ops when anon-tracking is off; without a real-delivery signal it would+ ;; look indistinguishable from a successful publish and the fingerprint would advance,+ ;; silently suppressing the boot-time retry once tracking is turned back on.+ (mt/with-temporary-setting-values [anon-tracking-enabled false]+ (let [result (complexity/complexity-scores :embedder nil)]+ (is (false? (:metabase-enterprise.data-complexity-score.complexity/snowplow-published?+ (meta result))))))))))diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0 b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0new file mode 100644index 00000000000..753f49dade4--- /dev/null+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/data_complexity/jsonschema/1-0-0@@ -0,0 +1,81 @@+{+ "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",+ "description": "Data Complexity Score — one event per (catalog, key) slice.",+ "self": {+ "vendor": "com.metabase",+ "name": "data_complexity",+ "format": "jsonschema",+ "version": "1-0-0"+ },+ "type": "object",+ "required": [+ "event",+ "catalog",+ "key",+ "score",+ "formula_version",+ "parameters"+ ],+ "properties": {+ "event": {+ "description": "Event name.",+ "type": "string",+ "enum": [+ "data_complexity_scoring"+ ],+ "maxLength": 64+ },+ "catalog": {+ "description": "Which catalog of the data layer was scored.",+ "type": "string",+ "enum": [+ "library",+ "universe",+ "metabot"+ ],+ "maxLength": 16+ },+ "key": {+ "description": "Dotted identifier for this slice: `total`, a `<group>.total` rollup, or a `<group>.<component>` leaf. Free-form so new keys can be added without a schema bump.",+ "type": "string",+ "maxLength": 64+ },+ "score": {+ "description": "Weighted score for this key. Null when the sub-score (or any sub-score it aggregates) couldn't be computed.",+ "type": [+ "integer",+ "null"+ ],+ "minimum": 0,+ "maximum": 2147483647+ },+ "measurement": {+ "description": "Raw pre-score measurement (absent on aggregates; null on uncomputed leaves). Number, not integer, to future-proof.",+ "type": [+ "number",+ "null"+ ],+ "minimum": 0,+ "maximum": 9007199254740991+ },+ "error": {+ "description": "Failure message on the originating leaf when its sub-score couldn't be computed; aggregates carry null `score` instead.",+ "type": [+ "string",+ "null"+ ],+ "maxLength": 1024+ },+ "formula_version": {+ "description": "Scoring formula version; bumps on breaking changes. Kept top-level so cross-version filters are a single column.",+ "type": "integer",+ "minimum": 1,+ "maximum": 2147483647+ },+ "parameters": {+ "description": "Scoring inputs that are likely to evolve (tunables, embedder identity). Sorted-key JSON so stable for SQL group-by; Metabase Transforms can break out its fields without bumping schema.",+ "type": "object",+ "additionalProperties": true+ }+ }+}diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 5165745d1f3..dcd4dcef4b3 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -400,6 +400,16 @@(prometheus/counter :metabase-search/semantic-indexer-dlq-failures{:description "Number of failed semantic search DLQ retries"})+ ;; data-complexity-score timing+ ;; 1ms → 1min buckets; widen later if real-world runs push past a minute.+ (prometheus/histogram :metabase-data-complexity/scoring-duration-ms+ {:description "Duration (ms) of a full Data Complexity Score run end-to-end (enumerate + score for all three catalogs + Snowplow publish)."+ :buckets [1 10 50 100 500 1000 5000 10000 30000 60000]})+ (prometheus/histogram :metabase-data-complexity/phase-duration-ms+ {:description "Duration (ms) of one stage (`enumerate` = DB fetch, `score` = in-memory scoring) for one catalog within a Data Complexity Score run."+ :labels [:stage :catalog]+ :buckets [1 10 50 100 500 1000 5000 10000 30000 60000]})+;; notification metrics(prometheus/counter :metabase-notification/send-ok{:description "Number of successful notification sends."diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.cljindex 1906b99d1d2..d4152c7c881 100644--- a/src/metabase/analytics/snowplow.clj+++ b/src/metabase/analytics/snowplow.clj@@ -54,7 +54,8 @@:snowplow/serialization "1-0-1":snowplow/simple_event "1-0-0":snowplow/cleanup "1-0-0"- :snowplow/ai_service_event "1-0-0"})+ :snowplow/ai_service_event "1-0-0"+ :snowplow/data_complexity "1-0-0"})(def ^:private SnowplowSchema"Malli enum for valid Snowplow schemas"@@ -144,20 +145,24 @@[tracker event](.track ^Tracker tracker ^SelfDescribing event))-(mu/defn track-event!+(mu/defn track-event! :- :boolean"Send a single analytics event to the Snowplow collector, if tracking is enabled for this MB instance and a collector- is available."+ is available. Returns true when the event was actually handed to the tracker; false when tracking is disabled or+ emission threw — callers that need to gate durable side-effects on real delivery can check the return value."([schema :- SnowplowSchema data](track-event! schema data api/*current-user-id*))([schema :- SnowplowSchema data user-id]- (when (analytics.settings/snowplow-enabled)- (try- (let [^SelfDescribing$Builder2 builder (-> (. SelfDescribing builder)- (.eventData (payload schema (schema->version schema) data))- (.customContext [(context)])- (cond-> user-id (.subject (subject user-id))))- ^SelfDescribing event (.build builder)]- (track-event-impl! tracker event))- (catch Throwable e- (log/errorf e "Error sending Snowplow analytics event for schema %s" schema))))))+ (boolean+ (when (analytics.settings/snowplow-enabled)+ (try+ (let [^SelfDescribing$Builder2 builder (-> (. SelfDescribing builder)+ (.eventData (payload schema (schema->version schema) data))+ (.customContext [(context)])+ (cond-> user-id (.subject (subject user-id))))+ ^SelfDescribing event (.build builder)]+ (track-event-impl! tracker event)+ true)+ (catch Throwable e+ (log/errorf e "Error sending Snowplow analytics event for schema %s" schema)+ false))))))diff --git a/src/metabase/collections/core.clj b/src/metabase/collections/core.cljindex c16a5e34850..16c916b38cd 100644--- a/src/metabase/collections/core.clj+++ b/src/metabase/collections/core.clj@@ -17,7 +17,9 @@create-library-collection!descendant-idslibrary-collection+ library-collection-typelibrary-data-collection-type+ library-metrics-collection-typelocation-pathmoving-from-remote-synced?moving-into-remote-synced?