Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use STS model for synonym detection (#73165)

Chris Truter01429bb7c516
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 07eafd11983..703f8e74065 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2866,6 +2866,7 @@
startup
task
util
+ enterprise/embeddings
enterprise/semantic-search}
:model-imports #{:model/Card
:model/Collection
@@ -2994,6 +2995,14 @@
:model/Table
:model/Tenant}}
+ enterprise/embeddings
+ {:team "Metabot"
+ :api #{metabase-enterprise.embeddings.client}
+ ;; Temporary: the impl currently delegates back into semantic-search until
+ ;; embeddings.client/get-embeddings-batch's TODO is executed (move impl + settings here,
+ ;; make token tracking a hook). Drop this dep once the relocation lands.
+ :uses #{enterprise/semantic-search}}
+
enterprise/gsheets
{:team "Cloud"
:api #{metabase-enterprise.gsheets.api
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
index 8ca1d5dbeb4..a2044207419 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/api.clj
@@ -3,6 +3,7 @@
(:require
[metabase-enterprise.data-complexity-score.complexity :as complexity]
[metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]))
@@ -75,7 +76,9 @@
(when-not (.compareAndSet api-scoring-running? false true)
(throw (ex-info "Data Complexity Score calculation already in progress" {:status-code 409})))
(try
- (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))
+ (complexity/complexity-scores
+ (assoc (synonym-source/complexity-scores-opts)
+ :metabot-scope (metabot-scope/internal-metabot-scope)))
(finally
(.set api-scoring-running? false))))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
index fa22e2d7f42..4f6f80db944 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -7,7 +7,6 @@
[clojure.pprint :as pprint]
[clojure.string :as str]
[metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
- [metabase-enterprise.semantic-search.core :as semantic-search]
[metabase.analytics-interface.core :as analytics.interface]
[metabase.analytics.core :as analytics]
[metabase.audit-app.core :as audit]
@@ -19,7 +18,11 @@
(set! *warn-on-reflection* true)
(def formula-version
- "Bump when the scoring formula changes in a way that would break historical comparisons."
+ "Bump when the scoring formula changes in a way that would break historical comparisons.
+
+ Changes to `embedding-model`, `synonym-threshold`, `text-variant`, etc don't need a bump.
+ These parameters are already captured in the fingerprint.
+ Formula versioning is reserved for cases where the actual formulas or components change."
1)
(def weights
@@ -44,10 +47,12 @@
(def synonym-similarity-threshold
"Cosine-similarity cutoff for flagging two names as synonyms.
- Higher than semantic-search's retrieval cutoff (0.30) because scoring needs precision, not recall.
- See https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb for how this
- value was chosen."
- 0.90)
+
+ Higher than semantic-search's retrieval cutoff (0.30) because scoring needs precision, not
+ recall. Note that we typically will use a STS model which typically produces lower similarity scores.
+
+ See https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb for background."
+ 0.80)
;;; ----------------------------------- enumeration -----------------------------------
;;;
@@ -345,18 +350,23 @@
[& parts]
(str/join "." (map snake parts)))
+(defn- snake-keys
+ "Walk maps recursively, snake-casing keyword keys and keyword leaf values to strings.
+ Sequential collections aren't traversed — current callers (`weights`, `embedding-model`) only
+ pass maps. Sorted-map output keeps the JSON serialization stable."
+ [x]
+ (cond
+ (map? x) (into (sorted-map) (map (fn [[k v]] [(snake k) (snake-keys v)])) x)
+ (keyword? x) (snake x)
+ :else x))
+
(defn- parameters-map
- "Sorted-map of scoring inputs likely to evolve, published as a JSON object on each event.
- String keys (top-level and nested) so they round-trip unchanged — Snowplow's `payload` only
- snake-cases top-level keys, and Cheshire would serialize nested keyword keys with their leading
- colon. Excludes `formula_version` — that stays top-level as the primary cross-version filter."
- [{:keys [synonym-threshold embedding-model weights]}]
+ "Sorted-map of scoring inputs likely to evolve, published as a JSON object on each event."
+ [{:keys [synonym-threshold embedding-model text-variant weights]}]
(cond-> (sorted-map "synonym_threshold" synonym-threshold
- "weights" (into (sorted-map)
- (map (fn [[k v]] [(snake k) v]))
- weights))
- embedding-model (assoc "embedding_model_provider" (:provider embedding-model)
- "embedding_model_name" (:model-name embedding-model))))
+ "weights" (snake-keys weights))
+ embedding-model (assoc "embedding_model" (snake-keys embedding-model))
+ text-variant (assoc "text_variant" (snake text-variant))))
(def ^:private max-error-length
"Matches the Snowplow schema's `error` maxLength — a pathological exception message
@@ -366,6 +376,14 @@
(defn- truncate-error [s]
(cond-> s (< max-error-length (count s)) (subs 0 max-error-length)))
+(defn- with-score
+ "Attach `:score` to `event` only when it is non-nil.
+ The `data_complexity` Snowplow schema flags `score` as non-nullable but optional, so an
+ uncomputed sub-score (or a rollup that cascaded nil from one) must omit the key entirely
+ rather than emit `\"score\": null`."
+ [event score]
+ (cond-> event (some? score) (assoc :score score)))
+
(defn- emit-snowplow!
"Submits Snowplow events for every score, every group aggregation, and the grand total.
Returns true when they are all successfully delivered.
@@ -377,23 +395,23 @@
events (for [[catalog result] [[:library library] [:universe universe] [:metabot metabot]]
event (concat (for [[component sub] (:components result)]
;; leaf component
- (cond-> (assoc base
- :catalog catalog
- :key (dotted-key (component->group component) component)
- :score (:score sub)
- :measurement (:measurement sub))
+ (cond-> (-> base
+ (assoc :catalog catalog
+ :key (dotted-key (component->group component) component)
+ :measurement (:measurement sub))
+ (with-score (:score sub)))
(:error sub) (assoc :error (truncate-error (:error sub)))))
(for [[group entries] (group-by #(component->group (key %)) (:components result))]
;; group total
- (assoc base
- :catalog catalog
- :key (dotted-key group :total)
- :score (nil-safe-sum (map (comp :score val) entries))))
+ (-> base
+ (assoc :catalog catalog
+ :key (dotted-key group :total))
+ (with-score (nil-safe-sum (map (comp :score val) entries)))))
;; grand total
- [(assoc base
- :catalog catalog
- :key (dotted-key :total)
- :score (:total result))])]
+ [(-> base
+ (assoc :catalog catalog
+ :key (dotted-key :total))
+ (with-score (:total result)))])]
event)]
;; No short-circuiting - even if they are failures, attempt the rest.
(reduce (fn [all-ok? event]
@@ -415,40 +433,40 @@
(defn complexity-scores
"Compute the complexity score for the `:library`, `:universe`, and `:metabot` catalogs of this
- Metabase instance. Returns a map of the shape:
-
- {:library {:total n :components {...}}
- :universe {:total n :components {...}}
- :metabot {:total n :components {...}}
- :meta {:formula-version 1
- :synonym-threshold 0.90
- :embedding-model {...}}}
-
- Options:
- `:embedder` — overrides the synonym-axis embedder (defaults to
- [[metabase-enterprise.semantic-search.core/search-index-embedder]]); pass `nil` to disable
- synonym scoring.
- `:metabot-scope` — a `{:verified-only? <bool> :collection-id <nil|Long>}` map describing how
- the internal Metabot filters Cards. `:metabot` is always scored separately from `:universe`
- because Metabot/search table visibility (hidden tables, routed databases) already diverges
- from the raw `:universe` set; the scope only narrows Cards further. The caller owns the
- scope decision (premium-feature gate + Metabot row lookup) so this namespace stays free of
- settings/feature/Metabot-row reads."
- [& {:keys [embedder metabot-scope] :as opts}]
- ;;; NOTE: we fully materialize a vector off all entities, along with one of those in the library, rather than
- ;;; returning reducibles. For very large instances that holds a non-trivial slim-entity list in memory
- ;;; (name, kind, field-count, measure-names — no fat columns like `result_metadata`),
- ;;; but each catalog is consumed by FIVE sub-score functions that each walk the collection, so making this
- ;;; reducible would re-query the app-db five times per scoring call — a worse tradeoff than the bounded memory
- ;;; we currently will currently consume (provided we have that memory).
+ Metabase instance.
+
+ Returns a map of the shape:
+
+ {:library {:total n :components {...}}
+ :universe {:total n :components {...}}
+ :metabot {:total n :components {...}}
+ :meta {:formula-version 1
+ :synonym-threshold 0.80
+ :embedding-model {:provider ..., :model-name ..., :model-dimensions ...}
+ :text-variant :names-split}}
+
+ Pure: this fn does not read settings or feature flags. Callers (api / task) resolve the
+ synonym-axis source via [[metabase-enterprise.data-complexity-score.synonym-source]] and pass
+ the result here.
+
+ Options:
+ `:embedder` — synonym-axis embedder. `nil` (or unset) disables synonym scoring.
+ `:embedding-model-meta` — `{:provider :model-name :model-dimensions}` published into
+ `:meta.embedding-model`. Pass nil to omit.
+ `:text-variant` — preprocessing variant published into `:meta.text-variant`. Pass nil
+ to omit (the search-index path passes nil because its preprocessing
+ isn't a single named variant).
+ `:metabot-scope` — `{:verified-only? <bool> :collection-id <nil|Long>}` describing how
+ the internal Metabot filters Cards."
+ [& {:keys [embedder embedding-model-meta text-variant metabot-scope]}]
+ ;;; NOTE: we fully materialize vectors of the relevant entities.
+ ;;; For very large instances that means holding large lists in memory, but each catalog is consumed
+ ;;; by many sub-score functions that each walk the collection, so making this reducible would
+ ;;; re-query the app-db five times per scoring call — a worse tradeoff than the bounded memory we
+ ;;; currently consume.
(let [total-timer (u/start-timer)]
(try
- (let [embedder (if (contains? opts :embedder)
- embedder
- semantic-search/search-index-embedder)
- model-meta (when (= embedder semantic-search/search-index-embedder)
- (semantic-search/active-embedding-model))
- {:keys [library universe metabot]}
+ (let [{:keys [library universe metabot]}
;; Single enumerate phase — see [[enumerate-catalogs]]. Library ⊆ universe and
;; metabot ⊆ universe, so fetching each catalog separately duplicated DB work; the
;; catalog label `"all"` on the enumerate stage reflects that one pass covers all
@@ -462,7 +480,8 @@
:metabot metabot-score
:meta (cond-> {:formula-version formula-version
:synonym-threshold synonym-similarity-threshold}
- model-meta (assoc :embedding-model model-meta))}]
+ embedding-model-meta (assoc :embedding-model embedding-model-meta)
+ text-variant (assoc :text-variant text-variant))}]
(log-scores! result)
(let [published? (try
(emit-snowplow! result)
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
index ce2cdfc04b0..c448c7eb35e 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj
@@ -1,9 +1,11 @@
(ns metabase-enterprise.data-complexity-score.complexity-embedders
"Pluggable embedding sources for the complexity score's synonym axis.
- An embedder takes entities `{:id :name :kind}` and returns `{normalized-name -> ^floats vector}`,
- omitting entities without a known vector."
+
+ An embedder takes entities `{:id :name :kind}` and returns
+ `{normalized-name -> ^floats vector}`, omitting entities without a known vector."
(:require
[clojure.string :as str]
+ [metabase-enterprise.embeddings.client :as embeddings]
[metabase.util :as u]))
(set! *warn-on-reflection* true)
@@ -13,12 +15,102 @@
[s]
(some-> s str/trim u/lower-case-en))
+(defn split-for-embedding
+ "Rewrite a raw name into space-separated natural-language tokens before embedding.
+
+ `_`, `-`, `.`, and camelCase boundaries become spaces; adjacent whitespace is collapsed and
+ the result is lowercased.
+ Splitting happens *before* lowercasing so the camelCase boundary stays visible.
+
+ Embedding models are trained on English: `\"monthly_active_users\"` is an out-of-distribution
+ token while `\"monthly active users\"` hits three well-understood ones.
+
+ See https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb for
+ calibration data."
+ [^String s]
+ (when s
+ (-> s
+ (str/replace #"([a-z])([A-Z])" "$1 $2")
+ (str/replace #"[_\-.]" " ")
+ (str/replace #"\s+" " ")
+ str/trim
+ u/lower-case-en)))
+
(defn fn-embedder
- "Build an embedder that delegates to a plain `(name-embed-fn names) -> [vectors]` function.
- Distinct normalized names are passed in; the returned vectors are zipped back by position.
+ "Build an embedder that delegates to `(name-embed-fn names) -> [vectors]`.
+
+ Distinct normalized names are passed in; returned vectors are zipped back by position.
Names whose `name-embed-fn` returns nil are omitted from the result map."
[name-embed-fn]
(fn embed [entities]
(let [names (->> entities (keep (comp normalize-name :name)) distinct vec)
vectors (when (seq names) (vec (name-embed-fn names)))]
(into {} (filter val) (zipmap names vectors)))))
+
+(def default-text-variant
+ "Which text form of each entity name gets embedded by the default synonym embedder.
+
+ `:names-split` rewrites snake/kebab/dotted/camelCase names into space-separated English
+ tokens before sending them to the provider — see [[split-for-embedding]].
+
+ Alternatives worth considering (not implemented): `:names` (raw lowercased name),
+ `:search-text` (type + name + description + schema, as the semantic-search indexer does),
+ or `:typed-split` ([source|value] prefix + split name).
+ The 2026-04-21 analysis shows names-split as the best default for both Arctic and MiniLM.
+
+ The value rides the fingerprint so a future swap to another variant forces a re-score
+ without a `formula-version` bump."
+ :names-split)
+
+(def ^:private ^Class floats-class
+ "`float[]` class object — cached so [[ensure-floats]] can `instance?`-check without
+ re-resolving on every vector."
+ (class (float-array 0)))
+
+(defn- ensure-floats
+ "Return `v` as a `float[]`, copying only when it isn't already one.
+
+ ai-service / openai providers already return primitive `float[]` (see `decode-embeddings`
+ in `metabase-enterprise.semantic-search.embedding`), so the common path skips the copy."
+ ^floats [v]
+ (if (instance? floats-class v) v (float-array v)))
+
+(def ^:private provider-batch-size
+ "Names per `get-embeddings-batch` HTTP call.
+
+ ai-service's `get-embeddings-batch` does no batching of its own, so without chunking here a
+ large catalog would land as a single multi-MB request/response. 256 keeps each call bounded
+ while staying well under any provider's per-request limit."
+ 256)
+
+(defn provider-embedder
+ "Build an embedder that embeds names via [[embeddings.client/get-embeddings-batch]] using
+ `model-descriptor` (`{:provider :model-name :model-dimensions}`).
+
+ For each distinct normalized name the raw form is sent through [[split-for-embedding]].
+ Splitting *before* lowercasing preserves the camelCase boundary (`\"pageViews\"` →
+ `\"page views\"`); the normalized key is what scoring looks up.
+
+ Texts are chunked into `provider-batch-size` pieces so a large catalog doesn't land as a
+ single oversized HTTP request — `get-embeddings-batch` does no chunking of its own for the
+ ai-service / openai providers.
+
+ Errors from the provider bubble up — `score-synonym-pairs` converts them into `nil`
+ measurements + an `:error` field so a broken run is visible downstream."
+ [model-descriptor]
+ (fn embed [entities]
+ (let [name->raw (reduce (fn [acc {nm :name}]
+ (if-let [n (normalize-name nm)]
+ (if (contains? acc n) acc (assoc acc n nm))
+ acc))
+ (array-map)
+ entities)
+ names (vec (keys name->raw))]
+ (when (seq names)
+ (let [texts (mapv #(split-for-embedding (get name->raw %)) names)
+ vectors (into []
+ (mapcat #(embeddings/get-embeddings-batch model-descriptor %))
+ (partition-all provider-batch-size texts))]
+ (into {}
+ (keep (fn [[n v]] (when v [n (ensure-floats v)])))
+ (map vector names vectors)))))))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj
index e1db149d62f..5768a10d7b3 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj
@@ -30,3 +30,49 @@
:type :string
:export? false
:doc false)
+
+(defsetting data-complexity-scoring-use-search-index-embedder
+ (deferred-tru
+ (str "Source the synonym-axis embeddings from the active semantic-search pgvector index "
+ "instead of calling the synonym-axis embedder configured below. The default (off) is "
+ "the better-calibrated signal — see "
+ "https://linear.app/metabase/document/synonym-analysis-21-april-2026-31c8ce76eddb "
+ "— but the search-index path does not require ai-service to be reachable, which is "
+ "useful for instances where it is not deployed."))
+ :encryption :no
+ :visibility :admin
+ :default false
+ :type :boolean
+ :export? false
+ :doc false)
+
+(defsetting data-complexity-scoring-synonym-embedding-provider
+ (deferred-tru "Provider used by the synonym-axis embedder when not falling back to the search index.")
+ :encryption :no
+ :visibility :admin
+ :default "ai-service"
+ :type :string
+ :export? false
+ :doc false)
+
+(defsetting data-complexity-scoring-synonym-embedding-model
+ (deferred-tru
+ (str "Model name passed to the synonym-axis embedding provider. Defaults to "
+ "sentence-transformers/all-MiniLM-L6-v2 — an STS-trained Sentence-Transformers model "
+ "calibrated against names-split preprocessing at threshold 0.80. See the synonym "
+ "calibration analysis before swapping this."))
+ :encryption :no
+ :visibility :admin
+ :default "sentence-transformers/all-MiniLM-L6-v2"
+ :type :string
+ :export? false
+ :doc false)
+
+(defsetting data-complexity-scoring-synonym-embedding-model-dimensions
+ (deferred-tru "Vector dimensions advertised for the synonym-axis embedding model.")
+ :encryption :no
+ :visibility :admin
+ :default 384
+ :type :positive-integer
+ :export? false
+ :doc false)
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/synonym_source.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/synonym_source.clj
new file mode 100644
index 00000000000..9ab6d1b786a
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/synonym_source.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.data-complexity-score.synonym-source
+ "Bridge from settings → synonym-axis embedder + the metadata to publish about it.
+
+ The complexity scoring code is intentionally settings-free (analysis modules take flags as opts).
+ This namespace is the only place that translates the synonym-related settings into the opts
+ callers splice into `complexity/complexity-scores`, and into the fingerprint fragment that lets
+ the cron skip re-scoring when nothing meaningful changed."
+ (:require
+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
+ [metabase-enterprise.data-complexity-score.settings :as settings]
+ [metabase-enterprise.semantic-search.core :as semantic-search]))
+
+(defn- configured-model-descriptor
+ "`{:provider :model-name :model-dimensions}` from the synonym-axis settings."
+ []
+ {:provider (settings/data-complexity-scoring-synonym-embedding-provider)
+ :model-name (settings/data-complexity-scoring-synonym-embedding-model)
+ :model-dimensions (settings/data-complexity-scoring-synonym-embedding-model-dimensions)})
+
+(defn complexity-scores-opts
+ "Map ready to splice into `complexity/complexity-scores`. When the search-index toggle is on,
+ routes the synonym axis through the active pgvector index and advertises whatever model that
+ index reports (nil when unreachable). Otherwise builds a fresh provider-embedder for the
+ configured descriptor and advertises the descriptor + the names-split text variant."
+ []
+ (if (settings/data-complexity-scoring-use-search-index-embedder)
+ {:embedder semantic-search/search-index-embedder
+ :embedding-model-meta (semantic-search/active-embedding-model)}
+ (let [descriptor (configured-model-descriptor)]
+ {:embedder (embedders/provider-embedder descriptor)
+ :embedding-model-meta descriptor
+ :text-variant embedders/default-text-variant})))
+
+(defn fingerprint-fragment
+ "Synonym-axis fragment of the scoring fingerprint. Toggling the source setting or swapping the
+ configured model forces a re-score; the search-index path also folds in the live active model
+ so a pgvector re-index that swaps the model invalidates prior scores."
+ []
+ (if (settings/data-complexity-scoring-use-search-index-embedder)
+ {:synonym-source :search-index
+ :embedding-model (or (semantic-search/active-embedding-model) :unavailable)}
+ {:synonym-source :default-provider
+ :embedding-model (configured-model-descriptor)
+ :text-variant embedders/default-text-variant}))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
index 4f75dd4461b..f2122b6864c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/task/complexity_score.clj
@@ -8,7 +8,7 @@
[metabase-enterprise.data-complexity-score.complexity :as complexity]
[metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
[metabase-enterprise.data-complexity-score.settings :as settings]
- [metabase-enterprise.semantic-search.core :as semantic-search]
+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase.app-db.cluster-lock :as cluster-lock]
[metabase.config.core :as config]
[metabase.task.core :as task]
@@ -22,16 +22,22 @@
(def ^:private trigger-key (triggers/key "metabase.task.data-complexity-score.trigger"))
(defn- current-fingerprint
- "String capturing everything that changes the meaning of an emitted score — mirror of the Snowplow
- `formula_version` + `parameters` fields. Includes `weights` so re-tuning forces a re-score
- without bumping `formula-version`; only structural changes to the scoring algorithm need that."
+ "String capturing everything that changes the meaning of an emitted score.
+
+ Mirrors the Snowplow `formula_version` + `parameters` fields. `weights` is included so re-tuning
+ forces a re-score without a `formula-version` bump; only structural scoring-algorithm changes
+ need that.
+
+ The synonym-axis fragment comes from [[synonym-source/fingerprint-fragment]] so the fingerprint
+ reacts to the source toggle and the configured model — and on the search-index path also picks
+ up swaps in the active pgvector model so a re-index that swaps the model invalidates prior
+ scores."
[]
- (let [embedding-model (semantic-search/active-embedding-model)]
- (pr-str (into (sorted-map)
- (cond-> {:formula-version complexity/formula-version
- :synonym-threshold complexity/synonym-similarity-threshold
- :weights complexity/weights}
- embedding-model (assoc :embedding-model embedding-model))))))
+ (pr-str (into (sorted-map)
+ (merge {:formula-version complexity/formula-version
+ :synonym-threshold complexity/synonym-similarity-threshold
+ :weights complexity/weights}
+ (synonym-source/fingerprint-fragment)))))
(defn- run-scoring!
"One scoring pass. Gated by [[settings/data-complexity-scoring-enabled]] so admins can silence
@@ -48,7 +54,9 @@
[claim-fingerprint]
(if (settings/data-complexity-scoring-enabled)
(try
- (let [result (complexity/complexity-scores :metabot-scope (metabot-scope/internal-metabot-scope))]
+ (let [result (complexity/complexity-scores
+ (assoc (synonym-source/complexity-scores-opts)
+ :metabot-scope (metabot-scope/internal-metabot-scope)))]
(if (::complexity/snowplow-published? (meta result))
(settings/data-complexity-scoring-last-fingerprint! claim-fingerprint)
(log/warn "Data Complexity Score: Snowplow publish failed; leaving fingerprint unchanged so the next boot or cron retries"))
diff --git a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
new file mode 100644
index 00000000000..607c81d3daf
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.embeddings.client
+ "Embedding-service RPC for callers outside the search engine.
+
+ This namespace is a thin neutral entry point so consumers like the data-complexity-score's
+ synonym axis can request embeddings without taking a transitive dependency on a
+ search-prefixed module for what is just an HTTP call to ai-service.
+
+ TODO: relocate the implementation here. Today the provider dispatch + HTTP impls (ollama,
+ ai-service, openai) and the embedding-service settings (`ee-embedding-service-base-url`,
+ `ee-embedding-service-api-key`, etc.) all live in `metabase-enterprise.semantic-search.*` —
+ they predate this namespace because semantic-search was the first consumer.
+
+ Steps for the move:
+ 1. Migrate the `get-embedding` / `get-embeddings-batch` / `pull-model` defmultis and
+ provider impls (currently in `metabase-enterprise.semantic-search.embedding`) into
+ `metabase-enterprise.embeddings.*`.
+ 2. Migrate the embedding-service settings into the same module (or a sibling
+ `metabase-enterprise.embeddings.settings` ns).
+ 3. Decouple the inline token-tracking calls (`semantic.models.token-tracking/record-tokens`,
+ `:metabase-search/semantic-embedding-tokens`, the snowplow `track-token-usage!`) — make
+ them a hookable callback so callers register their own analytics scope. Semantic-search
+ keeps recording its current metrics via that hook; data-complexity-score can opt out or
+ register its own.
+ 4. Flip the dependency: `semantic-search.embedding` becomes a thin re-export from this
+ namespace (or goes away entirely once consumers update).
+ 5. Search-index naming helpers (`abbrev-provider-name`, `abbrev-model-name`,
+ `clean-model-name`) and `get-configured-model` / `process-embeddings-streaming` stay in
+ semantic-search — they're search-engine-specific."
+ (:require
+ [metabase-enterprise.semantic-search.core :as semantic-search]))
+
+(set! *warn-on-reflection* true)
+
+(defn get-embeddings-batch
+ "Return a sequential collection of embedding vectors, in the same order as the input texts.
+
+ `embedding-model` — `{:provider :model-name :model-dimensions}` map; provider must be one of those registered on
+ [[metabase-enterprise.semantic-search.embedding/get-embeddings-batch]] (`ai-service`, `openai`,
+ `ollama`).
+ `texts` — sequential collection of input strings.
+ `opts` — optional keyword opts (e.g. `:type :doc`). Accepts alternating kwargs or a single trailing map;
+ forwarded as kwargs into the multimethod, which destructures with `& {:as opts}`."
+ [embedding-model texts & {:as opts}]
+ (apply semantic-search/get-embeddings-batch embedding-model texts (mapcat identity opts)))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 1b8f7736063..1460d07d6cf 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -5,6 +5,7 @@
[medley.core :as m]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedders]
+ [metabase-enterprise.semantic-search.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
@@ -22,7 +23,9 @@
(p/import-vars
[metabase-enterprise.semantic-search.embedders
active-embedding-model
- search-index-embedder])
+ search-index-embedder]
+ [metabase-enterprise.semantic-search.embedding
+ get-embeddings-batch])
(defn- fallback-engine
"Find the highest priority search engine available for fallback."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj
index 6558f6e49ca..13621700db2 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedders.clj
@@ -166,9 +166,15 @@
"Return the embedding model metadata for the *active* search index, not the current configuration.
Returns nil when the index is unreachable, not yet initialized, or the feature is disabled.
Use this instead of [[metabase-enterprise.semantic-search.env/get-configured-embedding-model]] when
- you need to know what model the index is *actually* serving — not just what the settings say."
+ you need to know what model the index is *actually* serving — not just what the settings say.
+
+ The returned map matches the shape produced by the synonym-axis settings in
+ [[metabase-enterprise.data-complexity-score.synonym-source]] so downstream consumers (Snowplow
+ payload, response :meta, scoring fingerprint) see one consistent embedding-model shape regardless
+ of whether the search-index path or a configured provider is in use."
[]
(when-let [{:keys [model]} (try-active-index-state)]
(when model
- {:provider (:provider model)
- :model-name (:model-name model)})))
+ {:provider (:provider model)
+ :model-name (:model-name model)
+ :model-dimensions (:vector-dimensions model)})))
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj
index 68f5b673bb2..e5ee00eb020 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/api_test.clj
@@ -3,7 +3,9 @@
[clojure.test :refer :all]
[metabase-enterprise.data-complexity-score.api :as api]
[metabase-enterprise.data-complexity-score.complexity :as complexity]
+ [metabase-enterprise.data-complexity-score.complexity-embedders :as embedders]
[metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase.metabot.config :as metabot.config]
[metabase.test :as mt]
[toucan2.core :as t2])
@@ -16,6 +18,18 @@
(def ^:private endpoint "ee/data-complexity-score/complexity")
+(defn- random-vec-for-name
+ "Deterministic 8-dim Gaussian vector seeded by the name's hash.
+ Same name always returns the same vector across catalogs, so library ⊆ universe synonym
+ pairs holds; different names produce visibly different vectors, so the synonym axis exercises
+ real cosine work instead of collapsing to zero."
+ ^floats [^String n]
+ (let [rng (java.util.Random. (long (hash n)))]
+ (float-array (repeatedly 8 #(.nextGaussian rng)))))
+
+(def ^:private random-synonym-embedder
+ (embedders/fn-embedder #(mapv random-vec-for-name %)))
+
(defn- internal-metabot-id
"Primary key of the internal Metabot row — used by the tests that temporarily tweak its
`use_verified_content`/`collection_id` via `mt/with-temp-vals-in-db`. Calls
@@ -34,40 +48,47 @@
(deftest complexity-endpoint-superuser-gets-consistent-totals-test
(testing "check invariants not covered by schema"
- (let [resp (mt/user-http-request :crowberto :get 200 endpoint)
- measurement (fn [cat k] (get-in resp [cat :components k :measurement]))
- component-score (fn [cat k] (get-in resp [cat :components k :score]))
- ;; NOTE: `:synonym-pairs` is intentionally included here even though it's *theoretically*
- ;; non-monotonic — `score-synonym-pairs` dedupes by normalized name and keeps whichever
- ;; embedding `search-index-embedder` happens to pick for that name, so adding
- ;; universe-only entities that collide on normalized name with a library entity could in
- ;; principle flip which vector wins and drop the pair count below library's. Reviewers
- ;; (human or AI) sometimes want to carve it out on that basis — don't. In every realistic
- ;; configuration (prod, dev, the fixture that backs this endpoint's hermetic path in
- ;; complexity_test.clj) the invariant holds, and asserting it keeps us honest about
- ;; regressions in the common case. If the edge case ever actually trips this, *that* is
- ;; the surprising thing we want to see and we'll deal with it then.
- component-keys [:entity-count :name-collisions :synonym-pairs
- :field-count :repeated-measures]]
- (testing ":total equals the sum of its component :score values"
- (doseq [catalog [:library :universe :metabot]
- :let [{:keys [total components]} (get resp catalog)]]
- (is (= total (reduce + (map :score (vals components))))
- (format "%s :total should equal sum of component :score values" catalog))))
- (testing "universe is a superset of library: every measurement and score ≥ library's"
- (doseq [k component-keys
- getter [measurement component-score]
- :let [lib (getter :library k)
- uni (getter :universe k)]]
- (is (>= uni lib)
- (format "universe %s (%s) should be ≥ library's (%s)" k uni lib))))
- (testing ":synonym-pairs can't exceed the number of distinct-name pairs possible"
- (doseq [catalog [:library :universe :metabot]
- :let [n-entities (measurement catalog :entity-count)
- syn-pairs (measurement catalog :synonym-pairs)
- max-pairs (/ (* n-entities (dec n-entities)) 2)]]
- (is (<= syn-pairs max-pairs)
- (format "%s :synonym-pairs (%s) can't exceed n*(n-1)/2 for n=%s" catalog syn-pairs n-entities)))))))
+ ;; Stub the synonym-source's opts to a deterministic hash-seeded random vector lookup. Returning
+ ;; {} would zero out the synonym axis and trivialize the invariants below; calling the real
+ ;; ai-service would make this test depend on environments where it's unreachable. Same name →
+ ;; same vector across catalogs preserves the library ⊆ universe pair invariant.
+ (mt/with-dynamic-fn-redefs [synonym-source/complexity-scores-opts
+ (constantly {:embedder random-synonym-embedder})]
+ (let [resp (mt/user-http-request :crowberto :get 200 endpoint)
+ measurement (fn [cat k] (get-in resp [cat :components k :measurement]))
+ component-score (fn [cat k] (get-in resp [cat :components k :score]))
+ ;; NOTE: `:synonym-pairs` is intentionally included here even though it's *theoretically*
+ ;; non-monotonic — `score-synonym-pairs` dedupes by normalized name and keeps whichever
+ ;; embedding the provider returns for that name, so adding universe-only entities that
+ ;; collide on normalized name with a library entity could in principle flip which vector
+ ;; wins and drop the pair count below library's. Reviewers (human or AI) sometimes want
+ ;; to carve it out on that basis — don't. In every realistic configuration (prod, dev,
+ ;; the fixture that backs this endpoint's hermetic path in complexity_test.clj) the
+ ;; invariant holds, and asserting it keeps us honest about regressions in the common
+ ;; case. If the edge case ever actually trips this, *that* is the surprising thing we
+ ;; want to see and we'll deal with it then.
+ component-keys [:entity-count :name-collisions :synonym-pairs
+ :field-count :repeated-measures]]
+ (testing ":total equals the sum of its component :score values"
+ (doseq [catalog [:library :universe :metabot]
+ :let [{:keys [total components]} (get resp catalog)
+ scores (map :score (vals components))]]
+ (is (= total (reduce + scores))
+ (format "%s :total should equal sum of component :score values" catalog))))
+ (testing "universe is a superset of library: every measurement and score ≥ library's"
+ (doseq [k component-keys
+ getter [measurement component-score]
+ :let [lib (getter :library k)
+ uni (getter :universe k)]]
+ (is (>= uni lib)
+ (format "universe %s (%s) should be ≥ library's (%s)" k uni lib))))
+ (testing ":synonym-pairs can't exceed the number of distinct-name pairs possible"
+ (doseq [catalog [:library :universe :metabot]
+ :let [n-entities (measurement catalog :entity-count)
+ syn-pairs (measurement catalog :synonym-pairs)
+ max-pairs (/ (* n-entities (dec n-entities)) 2)]]
+ (is (<= syn-pairs max-pairs)
+ (format "%s :synonym-pairs (%s) can't exceed n*(n-1)/2 for n=%s" catalog syn-pairs n-entities))))))))
(deftest complexity-endpoint-metabot-catalog-test
(testing ":metabot mirrors :universe when neither content-verification nor use_verified_content is active"
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index 54aa3d57a20..9bfde979ced 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -8,8 +8,11 @@
;; `scoring-task-registered-test` verifies init.clj's actual wiring path.
[metabase-enterprise.data-complexity-score.init]
[metabase-enterprise.data-complexity-score.metabot-scope :as metabot-scope]
- [metabase-enterprise.data-complexity-score.settings :as data-complexity-score.settings]
+ [metabase-enterprise.data-complexity-score.settings :as settings]
+ [metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
+ [metabase-enterprise.data-complexity-score.test-util :as test-util]
+ [metabase-enterprise.embeddings.client :as embeddings]
[metabase-enterprise.semantic-search.core :as semantic-search]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedders :as ss.embedders]
@@ -336,6 +339,10 @@
;; fetch-batch). Uses `:mock-indexed` so the *embedding model* is a lookup-table — but
;; the resulting vectors are real rows in the index table that search-index-embedder
;; queries via SQL.
+ ;;
+ ;; :meta.embedding-model is intentionally *not* asserted: the search-index embedder is no
+ ;; longer the default, and only the default path advertises a model descriptor. Passing it
+ ;; explicitly (as this test does) means the caller owns the model narrative, not us.
(when semantic.db.datasource/db-url
(let [captured (atom nil)]
(mt/with-premium-features #{:semantic-search}
@@ -343,8 +350,7 @@
(semantic.tu/with-test-db! {:mode :mock-indexed}
(reset! captured (complexity/complexity-scores
:embedder semantic-search/search-index-embedder)))))
- (is (=? {:meta {:embedding-model {:provider "mock" :model-name "model"}}
- :universe {:components {:synonym-pairs {:measurement number?
+ (is (=? {:universe {:components {:synonym-pairs {:measurement number?
:score nat-int?}}}}
@captured)
"embedder returned vectors from pgvector and the synonym axis produced a real measurement")))))
@@ -525,34 +531,83 @@
true? {:mid nil :model_id "abc" :model "card"} {:mid nil :model_id "xyz" :model "card"}
false? {:mid nil :model_id "xyz" :model "card"} {:mid nil :model_id "abc" :model "card"})))
-(deftest ^:sequential meta-embedding-model-absent-when-unavailable-test
+(deftest ^:sequential meta-passes-through-caller-supplied-model-and-variant-test
(mt/with-dynamic-fn-redefs [complexity/enumerate-catalogs
(constantly {:library [] :universe [] :metabot []})]
- (testing ":embedding-model key is absent from :meta when the search index is unreachable"
- (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state (constantly nil)]
- ;; Pass the real search-index-embedder so the identity check in complexity-scores succeeds,
- ;; but the embedder returns {} because try-active-index-state is nil.
- (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]
- (is (not (contains? meta :embedding-model))))))
- (testing ":embedding-model key is present in :meta when the active model is non-nil"
- (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
- (constantly {:pgvector :mock
- :table-name "mock_table"
- :model {:provider "openai"
- :model-name "text-embedding-3-small"}})
- ss.embedders/fetch-batch (constantly [])]
- (let [{:keys [meta]} (complexity/complexity-scores :embedder semantic-search/search-index-embedder)]
- (is (= {:provider "openai" :model-name "text-embedding-3-small"}
- (:embedding-model meta))))))))
+ (testing ":embedding-model and :text-variant are published in :meta when the caller passes them"
+ (let [{:keys [meta]} (complexity/complexity-scores
+ :embedder (embedders/fn-embedder (fn [_] []))
+ :embedding-model-meta {:provider "test" :model-name "fake" :model-dimensions 4}
+ :text-variant :names-split)]
+ (is (= {:provider "test" :model-name "fake" :model-dimensions 4} (:embedding-model meta)))
+ (is (= :names-split (:text-variant meta)))))
+ (testing ":embedding-model and :text-variant are omitted when the caller doesn't pass them"
+ (let [{:keys [meta]} (complexity/complexity-scores :embedder (embedders/fn-embedder (fn [_] [])))]
+ (is (not (contains? meta :embedding-model)))
+ (is (not (contains? meta :text-variant)))))
+ (testing ":embedding-model and :text-variant are omitted when synonym scoring is disabled (embedder is nil)"
+ (let [{:keys [meta]} (complexity/complexity-scores :embedder nil)]
+ (is (not (contains? meta :embedding-model)))
+ (is (not (contains? meta :text-variant)))))))
+
+(deftest ^:sequential synonym-source-default-opts-pin-minilm-via-ai-service-test
+ (testing "at default settings, synonym-source produces the MiniLM-L6-v2 ai-service descriptor + names-split text variant"
+ (test-util/with-synonym-source []
+ (let [{:keys [embedder embedding-model-meta text-variant]} (synonym-source/complexity-scores-opts)]
+ (is (= {:provider "ai-service"
+ :model-name "sentence-transformers/all-MiniLM-L6-v2"
+ :model-dimensions 384}
+ embedding-model-meta))
+ (is (= :names-split text-variant))
+ (is (fn? embedder)
+ "synonym-source returns a fresh provider-embedder for the descriptor")))))
+
+(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test
+ (testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"
+ (let [captured (atom nil)
+ embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 4})]
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
+ (fn [_model texts] (reset! captured (vec texts)) (repeat (count texts) [1.0]))]
+ (embedder
+ [{:id 1 :name "monthly_active_users" :kind :table}
+ {:id 2 :name "dim-date" :kind :table}
+ {:id 3 :name "pageViews" :kind :card}
+ {:id 4 :name "metabase.v2_foo" :kind :table}])
+ (is (= ["monthly active users"
+ "dim date"
+ "page views"
+ "metabase v2 foo"]
+ @captured))))))
+
+(deftest ^:sequential provider-embedder-propagates-provider-errors-test
+ (testing "provider errors bubble up so score-synonym-pairs can report nil measurements + :error"
+ (let [embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 4})]
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
+ (fn [_ _] (throw (ex-info "ai-service down" {})))]
+ (is (thrown-with-msg? Throwable #"ai-service down"
+ (embedder [{:id 1 :name "orders" :kind :table}])))))))
+
+(deftest ^:sequential provider-embedder-zips-by-position-test
+ (testing "vectors come back keyed by the normalized name; nil slots are dropped"
+ (let [embedder (embedders/provider-embedder {:provider "ai-service" :model-name "fake" :model-dimensions 2})]
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
+ (fn [_ texts]
+ (for [t texts] (when-not (= t "drop me") [1.0 0.0])))]
+ (let [result (embedder
+ [{:id 1 :name "keep me" :kind :table}
+ {:id 2 :name "drop_me" :kind :table}
+ {:id 3 :name "another" :kind :card}])]
+ (is (= #{"keep me" "another"} (set (keys result))))
+ (is (every? #(instance? (Class/forName "[F") %) (vals result))))))))
(deftest ^:sequential active-embedding-model-reads-from-active-index-test
(testing "active-embedding-model returns the model from the active index, not the configured setting"
- (let [active-model {:provider "openai" :model-name "text-embedding-ada-002"}]
+ (let [active-model {:provider "openai" :model-name "text-embedding-ada-002" :vector-dimensions 1536}]
(mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
(constantly {:pgvector :mock
:table-name "mock_table"
:model active-model})]
- (is (= {:provider "openai" :model-name "text-embedding-ada-002"}
+ (is (= {:provider "openai" :model-name "text-embedding-ada-002" :model-dimensions 1536}
(semantic-search/active-embedding-model))))))
(testing "active-embedding-model returns nil when the index state has no model"
(mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
@@ -673,11 +728,11 @@
(testing ":error is only present on the originating leaf — aggregates use null score instead"
(is (not-any? #(contains? % "error")
(vals (dissoc by-key "ambiguity.synonym_pairs")))))
- (testing "the failed leaf publishes null :score (no zero-fallback)"
- (is (nil? (get-in by-key ["ambiguity.synonym_pairs" "score"]))))
- (testing "aggregates that include the failed leaf cascade null :score"
- (is (nil? (get-in by-key ["ambiguity.total" "score"])))
- (is (nil? (get-in by-key ["total" "score"]))))
+ (testing "the failed leaf omits :score entirely (schema flags it non-nullable but optional, no zero-fallback)"
+ (is (not (contains? (get by-key "ambiguity.synonym_pairs") "score"))))
+ (testing "aggregates that include the failed leaf omit :score (cascade)"
+ (is (not (contains? (get by-key "ambiguity.total") "score")))
+ (is (not (contains? (get by-key "total") "score"))))
(testing "unaffected aggregates keep their numeric :score (cascade is leaf-scoped, not catalog-wide)"
;; size group has no synonym dependency, so its rollup must still be a real number even
;; when ambiguity falls through. Catches a regression where the cascade goes too far.
@@ -701,25 +756,23 @@
(is (= 1024 (count err))
"error is clipped to the schema's maxLength of 1024")))))))
-(deftest ^:sequential emit-snowplow-includes-embedding-model-meta-test
- (testing "every event's parameters carry embedding_model_provider/name when the search-index embedder is active"
+(deftest ^:sequential emit-snowplow-includes-embedding-model-and-text-variant-meta-test
+ (testing "every event's parameters carry the nested embedding_model + text_variant from synonym-source"
(snowplow-test/with-fake-snowplow-collector
- (mt/with-dynamic-fn-redefs [ss.embedders/try-active-index-state
- (constantly {:pgvector :mock
- :table-name "mock_table"
- :model {:provider "openai"
- :model-name "text-embedding-3-small"}})
- ss.embedders/fetch-batch (constantly [])
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch (fn [_ _] [])
complexity/enumerate-catalogs
(constantly {:library [(entity :name "orders")]
:universe [(entity :name "orders")]
:metabot []})]
(snowplow-test/pop-event-data-and-user-id!)
- (complexity/complexity-scores :embedder semantic-search/search-index-embedder)
- (let [events (complexity-events!)]
+ (complexity/complexity-scores (synonym-source/complexity-scores-opts))
+ (let [events (complexity-events!)
+ expected-model {"provider" "ai-service"
+ "model_name" "sentence-transformers/all-MiniLM-L6-v2"
+ "model_dimensions" 384}]
(is (seq events) "sanity: events were emitted")
- (is (every? #(= "openai" (get-in % ["parameters" "embedding_model_provider"])) events))
- (is (every? #(= "text-embedding-3-small" (get-in % ["parameters" "embedding_model_name"])) events)))))))
+ (is (every? #(= expected-model (get-in % ["parameters" "embedding_model"])) events))
+ (is (every? #(= "names_split" (get-in % ["parameters" "text_variant"])) events)))))))
(deftest ^:sequential emit-snowplow-failure-is-swallowed-test
(testing "emission failure is caught; complexity-scores still returns the score and logs a warning"
@@ -883,14 +936,14 @@
data-complexity-scoring-last-fingerprint "stale"]
(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]
(#'task.complexity-score/run-scoring! "fresh-fp")
- (is (= "fresh-fp" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (= "fresh-fp" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint stamped from the claim fingerprint, not re-sampled at commit time"))))
(testing "failed publish → fingerprint stays at the stale value for the next retry"
(mt/with-temporary-setting-values [data-complexity-scoring-enabled true
data-complexity-scoring-last-fingerprint "stale"]
(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]
(#'task.complexity-score/run-scoring! "fresh-fp")
- (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint preserved — next boot / cron will retry the emission")))))))
(deftest ^:sequential maybe-emit-boot-score-only-advances-fingerprint-on-successful-publish-test
@@ -904,9 +957,9 @@
data-complexity-scoring-claim ""]
(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result false))]
(task.complexity-score/maybe-emit-boot-score!)
- (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint unchanged — next boot/cron will retry the emission")
- (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))
+ (is (= "" (settings/data-complexity-scoring-claim))
"scoring claim released so other paths can proceed without waiting for TTL"))))
(testing "publish success → fingerprint advances to the new value (and claim is cleared)"
(mt/with-temporary-setting-values [data-complexity-scoring-enabled true
@@ -914,9 +967,9 @@
data-complexity-scoring-claim ""]
(mt/with-dynamic-fn-redefs [complexity/complexity-scores (fn [& _] (stub-result true))]
(task.complexity-score/maybe-emit-boot-score!)
- (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint advanced to reflect the confirmed publish")
- (is (= "" (data-complexity-score.settings/data-complexity-scoring-claim))
+ (is (= "" (settings/data-complexity-scoring-claim))
"scoring claim released after successful run")))))))
(deftest ^:sequential maybe-emit-boot-score-skips-when-another-path-holds-active-claim-test
@@ -937,9 +990,9 @@
(task.complexity-score/maybe-emit-boot-score!)
(is (false? @scoring-ran?)
"scoring skipped because another path already claimed the current fingerprint")
- (is (= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (= "stale" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint untouched when the claim is skipped")
- (is (= active-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ (is (= active-claim (settings/data-complexity-scoring-claim))
"other path's claim is preserved (we never took it, so we don't clear it)")))))))
(deftest ^:sequential maybe-emit-boot-score-reclaims-when-prior-claim-has-expired-test
@@ -960,7 +1013,7 @@
(task.complexity-score/maybe-emit-boot-score!)
(is (true? @scoring-ran?)
"scoring ran because the prior claim had aged past the TTL")
- (is (not= "stale" (data-complexity-score.settings/data-complexity-scoring-last-fingerprint))
+ (is (not= "stale" (settings/data-complexity-scoring-last-fingerprint))
"fingerprint advanced on successful publish after re-claim")))))))
(deftest ^:sequential maybe-emit-boot-score-does-not-clear-sibling-claim-after-ttl-takeover-test
@@ -979,10 +1032,10 @@
data-complexity-scoring-claim ""]
(mt/with-dynamic-fn-redefs [complexity/complexity-scores
(fn [& _]
- (data-complexity-score.settings/data-complexity-scoring-claim! sibling-claim)
+ (settings/data-complexity-scoring-claim! sibling-claim)
(stub-result true))]
(task.complexity-score/maybe-emit-boot-score!)
- (is (= sibling-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ (is (= sibling-claim (settings/data-complexity-scoring-claim))
"replacement claim preserved — our release was a compare-and-clear and the owners didn't match")))))))
(deftest ^:sequential cron-skips-when-boot-run-holds-active-claim-test
@@ -1006,7 +1059,7 @@
(#'task.complexity-score/with-scoring-claim! {} #'task.complexity-score/run-scoring!)
(is (false? @scoring-ran?)
"cron tick skipped because the boot run holds the scoring claim")
- (is (= boot-claim (data-complexity-score.settings/data-complexity-scoring-claim))
+ (is (= boot-claim (settings/data-complexity-scoring-claim))
"boot's claim preserved — cron never took it, so it doesn't clear it")))))))
(deftest ^:sequential complexity-scores-tags-publish-success-on-result-test
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/test_util.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/test_util.clj
new file mode 100644
index 00000000000..2f3f957e3e4
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/test_util.clj
@@ -0,0 +1,46 @@
+(ns metabase-enterprise.data-complexity-score.test-util
+ "Shared helpers for data-complexity-score tests."
+ (:require
+ [clojure.set :as set]
+ [metabase-enterprise.data-complexity-score.settings :as settings]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private setting-key
+ "Map shorthand → full setting symbol for [[with-synonym-source]]."
+ {:use-search-index? `settings/data-complexity-scoring-use-search-index-embedder
+ :provider `settings/data-complexity-scoring-synonym-embedding-provider
+ :model-name `settings/data-complexity-scoring-synonym-embedding-model
+ :model-dimensions `settings/data-complexity-scoring-synonym-embedding-model-dimensions})
+
+(def ^:private synonym-source-defaults
+ "Match the on-disk `:default` values in `settings.clj`. Keep in sync if those defaults move."
+ {:use-search-index? false
+ :provider "ai-service"
+ :model-name "sentence-transformers/all-MiniLM-L6-v2"
+ :model-dimensions 384})
+
+(defmacro with-synonym-source
+ "Pin every synonym-axis setting during `body` so tests don't depend on ambient state.
+
+ `overrides` is a flat vector of `:key value :key value` pairs. Recognized keys:
+ `:use-search-index?` — boolean; routes through the active pgvector index when true
+ `:provider` — provider string for the configured embedder path
+ `:model-name` — model name string
+ `:model-dimensions` — vector dimensions advertised for the model
+
+ Anything left out falls back to the same defaults as a fresh instance.
+ A bare `(with-synonym-source [] ...)` pretends nothing else has touched these settings."
+ [overrides & body]
+ (let [parsed (apply hash-map overrides)
+ unknown (set/difference (set (keys parsed)) (set (keys setting-key)))
+ _ (when (seq unknown)
+ (throw (ex-info (str "Unknown override keys: " unknown
+ ". Allowed: " (set (keys setting-key)))
+ {:unknown unknown})))
+ merged (merge synonym-source-defaults parsed)
+ bindings (vec (mapcat (fn [[short-key value]]
+ [(get setting-key short-key) value])
+ merged))]
+ `(mt/with-temporary-setting-values ~bindings ~@body)))