Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use the in-process embedder for retrieval and complexity score (#78314)

Chris Truter285d779b78a9
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 4ca5a7173ef..a89e54a4394 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -992,8 +992,8 @@
embeddings
{:team "Metabot"
- :api #{metabase.embeddings.provider}
- :uses #{util}}
+ :api #{metabase.embeddings.provider metabase.embeddings.startup}
+ :uses #{plugins util}}
entity-retrieval
{:team "Metabot"
@@ -3210,6 +3210,7 @@
collections
config
driver
+ embeddings
metabot
models
premium-features
@@ -3703,6 +3704,7 @@
collections
config
connection-pool
+ embeddings
events
health-inspector
lib-be
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 5be40dd0617..d162cd8fa53 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -13,7 +13,7 @@
:deprecated-var 84
:discouraged-java-method 3
:discouraged-namespace 80
- :discouraged-var 107
+ :discouraged-var 111
:equals-true 1
:main-without-gen-class 1
:metabase/defsetting-namespace 2
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 18f9330ffd0..d3d1440482b 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -232,7 +232,15 @@ embedder_plugin:
- "bin/build/src/build_embedder_plugin.clj"
- "deps.edn"
- "modules/embedder/**"
+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj"
+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj"
+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/synonym_source.clj"
+ - "enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj"
+ - "enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj"
+ - "enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj"
+ - "enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj"
- "src/metabase/embeddings/provider.clj"
+ - "src/metabase/embeddings/startup.clj"
- "src/metabase/plugins/**"
- "test/metabase/embeddings/embedder_plugin_artifact_test.clj"
diff --git a/.github/workflows/build-scripts.yml b/.github/workflows/build-scripts.yml
index 768770bce4c..2f87185af88 100644
--- a/.github/workflows/build-scripts.yml
+++ b/.github/workflows/build-scripts.yml
@@ -33,6 +33,15 @@ jobs:
timeout-minutes: 20
permissions:
contents: read
+ services:
+ postgres:
+ image: pgvector/pgvector:pg17
+ env:
+ POSTGRES_USER: mb_test
+ POSTGRES_DB: mb_test
+ POSTGRES_HOST_AUTH_METHOD: trust
+ ports:
+ - 5432:5432
steps:
- uses: actions/checkout@v6
- name: Prepare back-end environment
@@ -45,6 +54,8 @@ jobs:
jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase/embedder-model-catalog.edn
jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-arm64.zip
jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-avx2.zip
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/all-minilm-l6-v2-1110a243fdf4706b3f48f1d95db1a4f5529b4d41-arm64.zip
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/all-minilm-l6-v2-1110a243fdf4706b3f48f1d95db1a4f5529b4d41-avx2.zip
- name: Run artifact-only plugin smoke test
env:
MB_EMBEDDER_ARTIFACT_TEST: "true"
@@ -52,6 +63,19 @@ jobs:
run: >-
./bin/test-agent --oss :only
'[metabase.embeddings.embedder-plugin-artifact-test]'
+ - name: Run in-process semantic-search round trip
+ env:
+ MB_DB_TYPE: postgres
+ MB_DB_PORT: 5432
+ MB_DB_HOST: localhost
+ MB_DB_DBNAME: mb_test
+ MB_DB_USER: mb_test
+ MB_APPDB_PGVECTOR_MODE_TEST: "true"
+ MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST: "true"
+ MB_PLUGINS_DIR: modules/embedder/target
+ run: >-
+ ./bin/test-agent --drivers=postgres :only
+ '[metabase-enterprise.semantic-search.appdb-pgvector-mode-test/appdb-pgvector-in-process-embedder-round-trip-test]'
test-build-scripts:
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
diff --git a/bin/build/src/build/embedder_model.clj b/bin/build/src/build/embedder_model.clj
index 5ddcf03b11e..3aade9bd455 100644
--- a/bin/build/src/build/embedder_model.clj
+++ b/bin/build/src/build/embedder_model.clj
@@ -1,5 +1,5 @@
(ns build.embedder-model
- "Fetch, verify, and package the pinned Arctic model bundles used by the embedder plugin."
+ "Fetch, verify, and package the pinned model bundles used by the embedder plugin."
(:require
[clojure.edn :as edn]
[clojure.java.io :as io]
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
index f99fd8bb223..49e06eed433 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj
@@ -40,6 +40,7 @@
;; Loaded for side-effect: derives setting :on-change event topics from :metabase/event.
;; metabase.core.core/entrypoint normally does this, but the standalone CLI bypasses it.
[metabase.driver.init]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -132,6 +133,9 @@
Snowplow is off here, so we don't advance `data-complexity-scoring-last-fingerprint` — leave that to the cron."
[write?]
(mdb/setup-db-without-migrations!)
+ ;; `metabase.core.core/init!` does this on the server, but standalone modes skip it. Without it a synonym axis
+ ;; configured for the `in-process` provider finds nothing registered and fails with `:provider-not-registered`.
+ (embeddings.startup/ensure-in-process-provider!)
(let [result (complexity/complexity-scores
(assoc (synonym-source/complexity-scores-opts)
:metabot-scope (metabot-scope/internal-metabot-scope)
diff --git a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
index bb6cf1a547d..cdc96ac8a90 100644
--- a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
@@ -31,22 +31,23 @@
(set! *warn-on-reflection* true)
-(defn- search-embedding-model
+(defn- normalize-model-descriptor
"Translate the neutral embedding descriptor's dimension key to semantic search's internal key."
- [embedding-model]
- (cond-> embedding-model
- (and (nil? (:vector-dimensions embedding-model))
- (some? (:model-dimensions embedding-model)))
- (assoc :vector-dimensions (:model-dimensions embedding-model))))
+ [{:keys [model-dimensions vector-dimensions] :as embedding-model}]
+ (when (and model-dimensions vector-dimensions (not= model-dimensions vector-dimensions))
+ (throw (ex-info "Embedding model descriptor has conflicting dimension values."
+ {:model-dimensions model-dimensions
+ :vector-dimensions vector-dimensions})))
+ (cond-> (dissoc embedding-model :model-dimensions)
+ model-dimensions (assoc :vector-dimensions model-dimensions)))
(defn get-embeddings-batch
"Return a sequential collection of embedding vectors, in the same order as the input texts.
`embedding-model` — `{:provider :model-name :model-dimensions}` map; provider must be one of those registered on
- [[metabase-enterprise.semantic-search.embedding/get-embeddings-batch]] (`ai-service`, `openai`,
- `ollama`).
+ [[metabase.embeddings.provider]] (`ai-service`, `openai`, `ollama`).
`texts` — sequential collection of input strings.
`opts` — optional keyword opts (e.g. `:type :doc`). Accepts alternating kwargs or a single trailing map;
- forwarded as kwargs into the multimethod, which destructures with `& {:as opts}`."
+ forwarded as kwargs to the provider."
[embedding-model texts & {:as opts}]
- (apply semantic-search/get-embeddings-batch (search-embedding-model embedding-model) texts (mapcat identity opts)))
+ (apply semantic-search/get-embeddings-batch (normalize-model-descriptor embedding-model) texts (mapcat identity opts)))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
index c4e14f6a3ab..7c52699ebc4 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
@@ -127,6 +127,22 @@
(pgvector-configured?)
(embedder-configured?)))
+(defn- resolved-configured-model
+ "Resolve the global setting to the immutable embedding space used by persisted library vectors."
+ []
+ (embedding/resolve-model (embedding/get-configured-model)))
+
+(defn- resolved-configured-model-for-search
+ []
+ (try
+ (resolved-configured-model)
+ (catch clojure.lang.ExceptionInfo e
+ ;; Readiness can change after the tool is offered (for example settings can select an unavailable model).
+ ;; Domain/configuration failures should make this optional retrieval path disappear, while JVM errors and
+ ;; other unexpected exception types still escape.
+ (log/warn e "configured embedding model could not be resolved; returning no library results")
+ nil)))
+
(defn- missing-table-error?
"Whether we threw due to a Postgres undefined_table error (42P01), i.e. the indexes don't exist yet.
Expected before the first build or after a manual drop awaiting heal.
@@ -154,7 +170,7 @@
[probe-populated?]
(try
(let [ds (semantic.db.datasource/ensure-initialized-data-source!)
- metadata-status (index-table/index-status ds (embedding/get-configured-model))
+ metadata-status (index-table/index-status ds (resolved-configured-model))
status (if (and (= :compatible metadata-status)
(not (index-table/vectors-table-exists? ds)))
:missing
@@ -277,7 +293,7 @@
(let [ds (semantic.db.datasource/ensure-initialized-data-source!)
waited-ms (elapsed-ms scheduled)
started (u/start-timer)
- diff (reconcile/reconcile! ds embedding/get-configured-model)
+ diff (reconcile/reconcile! ds resolved-configured-model)
ran-ms (elapsed-ms started)]
(record-run! "full" diff ran-ms)
{:index (select-keys diff [:inserted :deleted :unchanged])
@@ -296,7 +312,7 @@
(doseq [[entity-type entity-local-id :as entity-key] dirty]
(try
(let [started (u/start-timer)
- diff (reconcile/reconcile-entity! ds embedding/get-configured-model entity-type entity-local-id)]
+ diff (reconcile/reconcile-entity! ds resolved-configured-model entity-type entity-local-id)]
(record-run! "targeted" diff (elapsed-ms started)))
(catch Throwable e
(log/error "library entity index: targeted reconcile failed; re-queuing"
@@ -373,6 +389,15 @@
(format "WHEN doc_type = '%s' THEN %s" (name k) w)))]
(format "(%s) - (CASE %s ELSE 0.0 END)" distance-expr cases)))
+(defn- index-compatible?
+ [connectable model]
+ (try
+ (= :compatible (index-table/index-status connectable model))
+ (catch SQLException e
+ (if (= "42P01" (.getSQLState e))
+ false
+ (throw e)))))
+
(defenterprise search-unfiltered
"Find the library-entity documents best matching `user-search-prompt`, up to `limit`, ranked by a
blended score (cosine similarity plus a slight doc_type bump).
@@ -388,42 +413,57 @@
[]
(let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
limit (or limit default-limit)
- model (embedding/get-configured-model)
- embedding (embedding/get-embedding model user-search-prompt
- {:type :query :record-tokens? true})
- lit (index-table/format-embedding embedding)
- distance (str "doc_embedding <=> " lit)
- rows (try
- (jdbc/execute!
- pgvector
- (-> (sql.helpers/select :entity_type :entity_local_id :doc_type :doc_text
- [[:raw distance] :distance])
- (sql.helpers/from (keyword (index-table/vectors-table)))
- ;; Exact scan, no HNSW: the blended order-by can't use an ANN index, and the
- ;; library set is tiny.
- (sql.helpers/order-by [[:raw (ranking-sql distance)] :asc])
- (sql.helpers/limit limit)
- (sql/format {:quoted true}))
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (catch SQLException e
- ;; Only the index-not-ready states degrade to no results — the index, not the agent,
- ;; is at fault and the next reconcile heals it:
- ;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet)
- ;; 22000 stored vectors incompatible with the query vector (a dimension/format change
- ;; awaiting rebuild — e.g. a vector(OLD) column vs a vector(NEW) literal)
- ;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool
- ;; reports the search as unavailable rather than as an empty library.
- (case (.getSQLState e)
- "42P01" []
- "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))
- [])
- (do (analytics/inc! :metabase-entity-retrieval/search-failed)
- (throw e)))))]
- (->> rows
- (map (fn [row]
- {:entity {:model (:entity_type row) :id (:entity_local_id row)}
- :doc_type (:doc_type row)
- :doc_text (:doc_text row)
- :score (score row)}))
- (sort-by (comp :total_score :score) >)
- vec))))
+ model (resolved-configured-model-for-search)]
+ ;; Check compatibility under a short non-blocking read lock. Even metadata-table DDL can otherwise block this
+ ;; preflight behind reconcile before we reach the protected query path. The later locked recheck remains
+ ;; authoritative because a reconcile can begin while the embedding is generated.
+ (if-not (and model
+ (true? (reconcile/with-index-read-lock pgvector #(index-compatible? % model))))
+ []
+ (let [embedding (embedding/get-embedding model (embedding/prefix-search-query model user-search-prompt)
+ {:type :query :record-tokens? true})
+ lit (index-table/format-embedding embedding)
+ distance (str "doc_embedding <=> " lit)]
+ (or
+ (reconcile/with-index-read-lock
+ pgvector
+ (fn [conn]
+ ;; The compatibility check and query share the reconcile lock and connection. A setting change
+ ;; after the tool was offered must never query same-width vectors from another immutable space.
+ (if-not (index-compatible? conn model)
+ []
+ (let [rows (try
+ (jdbc/execute!
+ conn
+ (-> (sql.helpers/select :entity_type :entity_local_id :doc_type :doc_text
+ [[:raw distance] :distance])
+ (sql.helpers/from (keyword (index-table/vectors-table)))
+ ;; Exact scan, no HNSW: the blended order-by can't use an ANN index, and the
+ ;; library set is tiny.
+ (sql.helpers/order-by [[:raw (ranking-sql distance)] :asc])
+ (sql.helpers/limit limit)
+ (sql/format {:quoted true}))
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (catch SQLException e
+ ;; Only the index-not-ready states degrade to no results — the index, not the agent,
+ ;; is at fault and the next reconcile heals it:
+ ;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet)
+ ;; 22000 stored vectors incompatible with the query vector (a dimension/format change
+ ;; awaiting rebuild — e.g. a vector(OLD) column vs a vector(NEW) literal)
+ ;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool
+ ;; reports the search as unavailable rather than as an empty library.
+ (case (.getSQLState e)
+ "42P01" []
+ "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))
+ [])
+ (do (analytics/inc! :metabase-entity-retrieval/search-failed)
+ (throw e)))))]
+ (->> rows
+ (map (fn [row]
+ {:entity {:model (:entity_type row) :id (:entity_local_id row)}
+ :doc_type (:doc_type row)
+ :doc_text (:doc_text row)
+ :score (score row)}))
+ (sort-by (comp :total_score :score) >)
+ vec)))))
+ []))))))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
index 022c2c85bcb..2f3b4d52150 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
@@ -52,7 +52,7 @@
:meta (str index-schema ".library_entity_index_meta")})
(def legacy-tables
- "The unqualified names used before [[index-schema]] existed. [[adopt-legacy-tables!]] moves them."
+ "The unqualified names used before [[index-schema]] and immutable embedding-space identities existed."
{:vectors "library_entity_index"
:meta "library_entity_index_meta"})
@@ -88,17 +88,17 @@
(semantic.util/quote-table (meta-table)))
(def schema-version
- "Canonical version of the index's *document format* — both the vectors table schema and the
+ "Canonical version of the persisted index contract — the metadata and vectors table schemas plus the
doc-derivation contract (doc_id scheme, doc_type set, doc_text source, dedup/key rules).
It's part of the meta row's [[model-identity]], so bumping it makes [[ensure-tables!]] drop and rebuild
the vectors table; the post-upgrade startup reconcile then repopulates from the appdb under the new
- format. Bump on ANY format-affecting change in [[metabase-enterprise.entity-retrieval.reconcile]] or
- the table schema: a vectors-table column/type change, a new or renamed doc_type, a changed doc_text
+ format. Bump on ANY compatibility-affecting change in [[metabase-enterprise.entity-retrieval.reconcile]]
+ or either table schema: a column/type change, a new or renamed doc_type, a changed doc_text
source, or a changed doc_id / dedup / key scheme — anything that makes old rows incomparable to newly
derived desired docs. A bump forces a full re-embed of the library on every instance at upgrade, so do
it only when the format truly moved, never as a refresh convenience."
- ;; v1 — initial schema.
- 1)
+ ;; v1 — initial schema; v2 — immutable embedding-space identity.
+ 2)
;; Advisory lock serializing concurrent ensure-tables! calls (e.g. several cluster nodes starting at
;; once). Arbitrary app-wide-unique constant; semantic-search's migration lock uses 19991.
@@ -124,6 +124,7 @@
[:provider :text :not-null]
[:model_name :text :not-null]
[:vector_dimensions :int :not-null]
+ [:embedding_space_id :text :not-null]
[:schema_version :int :not-null]
[:updated_at :timestamp-with-time-zone :not-null]
;; Captured immediately before a successful full reconcile reads the appdb. Distinct from updated_at
@@ -149,27 +150,45 @@
{:provider (:provider embedding-model)
:model_name (:model-name embedding-model)
:vector_dimensions (:vector-dimensions embedding-model)
+ :embedding_space_id (:embedding-space-id embedding-model)
:schema_version schema-version})
-(defn- read-meta [tx]
- (jdbc/execute-one! tx
- [(format "SELECT provider, model_name, vector_dimensions, schema_version FROM %s WHERE id = 1"
- (meta-table-sql))]
- {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+(defn- meta-column-exists? [tx column]
+ (some? (jdbc/execute-one! tx [(str "SELECT 1 FROM pg_attribute"
+ " WHERE attrelid = to_regclass(?) AND attname = ? AND NOT attisdropped")
+ (meta-table) column])))
+
+(defn- read-meta
+ "The meta row shaped like [[model-identity]], or nil when nothing is built.
+
+ A v1 table has no `embedding_space_id` column. Report it as nil instead of failing the SELECT, so the
+ identity comparison in [[index-status]] / [[ensure-tables!]] sees the mismatch and schedules the rebuild
+ that fills it in."
+ [tx]
+ (let [space? (meta-column-exists? tx "embedding_space_id")]
+ (some->> (jdbc/execute-one! tx
+ [(format (str "SELECT provider, model_name, vector_dimensions, schema_version%s"
+ " FROM %s WHERE id = 1")
+ (if space? ", embedding_space_id" "")
+ (meta-table-sql))]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (merge {:embedding_space_id nil}))))
(defn- write-meta! [tx embedding-model]
- (let [{:keys [provider model_name vector_dimensions schema_version]} (model-identity embedding-model)]
+ (let [{:keys [provider model_name vector_dimensions embedding_space_id schema_version]}
+ (model-identity embedding-model)]
(jdbc/execute! tx
(-> (sql.helpers/insert-into (keyword (meta-table)))
(sql.helpers/values [{:id 1
:provider provider
:model_name model_name
:vector_dimensions vector_dimensions
+ :embedding_space_id embedding_space_id
:schema_version schema_version
:updated_at (Instant/now)}])
(sql.helpers/on-conflict :id)
(sql.helpers/do-update-set :provider :model_name :vector_dimensions
- :schema_version :updated_at)
+ :embedding_space_id :schema_version :updated_at)
sql-format-quoted))))
(defn- create-tables! [tx dims]
@@ -196,36 +215,45 @@
e))))))
;; TODO (Chris 2026-08-17) -- delete this once no supported upgrade can start from the bare names.
-(defn- adopt-legacy-tables!
- "Move an index built before [[index-schema]] existed into it, preserving its rows.
- Without this, the qualified tables would be created empty beside the old ones and the whole library
- would be re-embedded. A no-op once moved: the unqualified name no longer resolves.
-
- A dedicated store only. The bare names date from when this index was dedicated-only, so nothing in an
- app db can be an old index of ours -- but [[table-exists?]] resolves an unqualified name on the search
- path, where an application table that happens to share the name would answer, and get moved."
+(defn- legacy-table-in-search-path
+ "The first table named `table-name` on the effective search path, excluding the current index schema."
+ [tx table-name]
+ (when-let [schema-name (:schema_name
+ (jdbc/execute-one!
+ tx
+ [(str "SELECT n.nspname AS schema_name"
+ " FROM unnest(current_schemas(true)) WITH ORDINALITY AS p(schema_name, search_order)"
+ " JOIN pg_namespace n ON n.nspname = p.schema_name"
+ " JOIN pg_class c ON c.relnamespace = n.oid"
+ " WHERE c.relname = ? AND c.relkind IN ('r', 'p') AND n.nspname <> ?"
+ " ORDER BY p.search_order LIMIT 1")
+ table-name index-schema]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))]
+ {:schema-name schema-name :table-name table-name}))
+
+(defn- drop-legacy-tables!
+ "Drop an index built before [[index-schema]] and immutable embedding-space identities existed.
+
+ Its vectors have no trustworthy space identity, so they must be rebuilt rather than moved into the
+ current schema. A dedicated store only: an app db may contain unrelated application tables with these
+ names. Resolve the effective search path, excluding the current index schema, then drop the resolved
+ tables by qualified name."
[tx]
(when (and (= default-tables (tables))
(semantic.db.datasource/dedicated-url-configured?))
(doseq [k [:vectors :meta]]
- (let [legacy (k legacy-tables)]
- (when (and (table-exists? tx legacy)
- (not (table-exists? tx (k default-tables))))
- (log/infof "Moving %s into the %s schema" legacy index-schema)
- (jdbc/execute! tx [(format "ALTER TABLE %s SET SCHEMA %s"
- (semantic.util/quote-ident legacy)
- (semantic.util/quote-ident index-schema))]))))))
+ (when-let [{:keys [schema-name table-name]} (legacy-table-in-search-path tx (k legacy-tables))]
+ (let [legacy-sql (format "%s.%s"
+ (semantic.util/quote-ident schema-name)
+ (semantic.util/quote-ident table-name))]
+ (log/infof "Dropping incompatible legacy library entity index table %s.%s" schema-name table-name)
+ (jdbc/execute! tx [(format "DROP TABLE %s" legacy-sql)]))))))
(defn vectors-table-exists?
"Whether the configured entity-retrieval vectors table exists."
[tx]
(table-exists? tx (vectors-table)))
-(defn- meta-column-exists? [tx column]
- (some? (jdbc/execute-one! tx [(str "SELECT 1 FROM pg_attribute"
- " WHERE attrelid = to_regclass(?) AND attname = ? AND NOT attisdropped")
- (meta-table) column])))
-
(defn- can-alter-meta-table? [tx]
(:owns_table
(jdbc/execute-one! tx
@@ -271,6 +299,38 @@
(jdbc/execute! tx [(format "UPDATE %s SET reconciled_at = NULL WHERE id = 1"
(meta-table-sql))])))
+(defn- embedding-space-column-state
+ "Whether the meta table's `embedding_space_id` column is `:missing`, `:nullable`, or `:not-null`."
+ [tx]
+ (if-let [row (jdbc/execute-one! tx
+ [(str "SELECT attnotnull FROM pg_attribute"
+ " WHERE attrelid = to_regclass(?) AND attname = 'embedding_space_id'"
+ " AND NOT attisdropped")
+ (meta-table)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (if (:attnotnull row) :not-null :nullable)
+ :missing))
+
+(defn- ensure-embedding-space-column!
+ "Add `embedding_space_id` to a meta table built before it existed. Returns the column's prior state.
+
+ Unlike `reconciled_at`, this column is load-bearing: [[read-meta]] reads it and [[model-identity]]
+ compares it, so a role that cannot add it can never bring the index to a compatible state. Raise a typed
+ error naming the ownership problem rather than letting a bare SQLSTATE 42501 abort the caller's
+ transaction — [[metabase-enterprise.entity-retrieval.core/retrieval-status]] reports it as `:unreachable`
+ and the retrieval tool stops being offered until an admin fixes the grant."
+ [tx]
+ (let [state (embedding-space-column-state tx)]
+ (when (= :missing state)
+ (when-not (can-alter-meta-table? tx)
+ (throw (ex-info (format (str "Cannot add embedding_space_id to %s: the current role does not own it."
+ " Grant ownership of the library entity index tables to the Metabase"
+ " user, or drop them so they can be recreated.")
+ (meta-table))
+ {:type ::embedding-space-upgrade-failed, :table (meta-table)})))
+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN embedding_space_id TEXT" (meta-table-sql))]))
+ state))
+
(defn index-status
"Compatibility of the built index against `embedding-model` and [[schema-version]]:
- `:missing` no meta row — nothing built
@@ -299,41 +359,49 @@
(jdbc/execute! tx [(format "SELECT pg_advisory_xact_lock(%d)" ensure-lock-id)])
(jdbc/execute! tx (sql/format (sql.helpers/create-extension :vector :if-not-exists)))
(ensure-schema! tx)
- ;; Before any CREATE below: those are IF NOT EXISTS, so creating first would leave an empty qualified
- ;; table for the move to refuse, and the whole library would be re-embedded.
- (adopt-legacy-tables! tx)
+ ;; Before any CREATE below: remove the pre-embedding-space tables whose vectors cannot be trusted.
+ (drop-legacy-tables! tx)
(jdbc/execute! tx (create-meta-table-sql))
;; CREATE TABLE IF NOT EXISTS does not add columns to an existing table. Upgrade when this role owns it;
;; grant-only roles keep reconciling without the optional freshness timestamp.
(ensure-reconciled-at-column! tx)
- (let [stored (read-meta tx)
- current (model-identity embedding-model)
- dims (:vector_dimensions current)
- result (cond
- (nil? stored)
- (do (create-tables! tx dims)
- (write-meta! tx embedding-model)
- :created)
-
- (not= stored current)
- (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS %s"
- (vectors-table-sql))])
- (create-tables! tx dims)
- (write-meta! tx embedding-model)
- :rebuilt)
-
- :else
- ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table
- ;; heals itself, and report :created when it had actually gone missing — the recreated
- ;; table is empty, so a targeted reconcile must repopulate the whole library rather than
- ;; fill it with one entity.
- (let [existed? (vectors-table-exists? tx)]
- (create-tables! tx dims)
- (if existed? :ok :created)))]
+ (let [space-column (ensure-embedding-space-column! tx)
+ stored (read-meta tx)
+ current (model-identity embedding-model)
+ dims (:vector_dimensions current)
+ result (cond
+ (nil? stored)
+ (do (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :created)
+
+ (not= stored current)
+ (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS %s"
+ (vectors-table-sql))])
+ (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :rebuilt)
+
+ :else
+ ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table
+ ;; heals itself, and report :created when it had actually gone missing — the recreated
+ ;; table is empty, so a targeted reconcile must repopulate the whole library rather
+ ;; than fill it with one entity.
+ (let [existed? (vectors-table-exists? tx)]
+ (create-tables! tx dims)
+ (if existed? :ok :created)))]
;; Any empty (re)build invalidates reconciled_at: the table is empty until a full reconcile repopulates
;; it, and neither write-meta! (:rebuilt) nor the heal branch touches reconciled_at, so a prior build's
;; timestamp would linger and make NLQ staleness read fresh over an empty/incomplete index. Clear it;
;; touch-reconciled-at! (converged reconciles only) sets it again once the index is actually verified.
(when (not= result :ok)
(clear-reconciled-at! tx))
+ ;; Every row carries a space id by now — the mismatch above forced the rebuild that wrote it — so the
+ ;; constraint can go on. SET NOT NULL takes an ACCESS EXCLUSIVE lock and needs ownership, so skip it
+ ;; when it is already there or the role cannot alter the table; the identity compare, not the
+ ;; constraint, is what keeps a stale index from being served.
+ (when (and (not= :not-null space-column)
+ (can-alter-meta-table? tx))
+ (jdbc/execute! tx [(format "ALTER TABLE %s ALTER COLUMN embedding_space_id SET NOT NULL"
+ (meta-table-sql))]))
result)))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
index 54af8e03d50..3621b37da52 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
@@ -50,6 +50,47 @@
;; semantic-search's migration lock (19991).
(def ^:private reconcile-lock-id 20012)
+(defn- with-index-lock
+ [pgvector lock-function unlock-function f]
+ (with-open [^Connection conn (jdbc/get-connection pgvector)]
+ (let [autocommit (.getAutoCommit conn)]
+ (.setAutoCommit conn true)
+ (try
+ (jdbc/execute! conn [(format "SELECT %s(%d)" lock-function reconcile-lock-id)])
+ (try
+ (f conn)
+ (finally
+ (jdbc/execute! conn [(format "SELECT %s(%d)" unlock-function reconcile-lock-id)])))
+ (finally
+ (.setAutoCommit conn autocommit))))))
+
+(defn with-index-read-lock
+ "Run `(f connection)` under a shared cluster-wide library-index lock, returning nil immediately when a
+ reconcile holds the matching exclusive lock.
+
+ Concurrent searches share this lock. The non-blocking acquisition prevents waiting searches from exhausting
+ the pgvector connection pool during a long reconcile. A successful search's compatibility check and vector
+ query cannot straddle a rebuild into another embedding space."
+ [pgvector f]
+ (with-open [^Connection conn (jdbc/get-connection pgvector)]
+ (let [autocommit (.getAutoCommit conn)]
+ (.setAutoCommit conn true)
+ (try
+ (when (:acquired (jdbc/execute-one!
+ conn
+ [(format "SELECT pg_try_advisory_lock_shared(%d) AS acquired" reconcile-lock-id)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+ (try
+ (f conn)
+ (finally
+ (jdbc/execute! conn [(format "SELECT pg_advisory_unlock_shared(%d)" reconcile-lock-id)]))))
+ (finally
+ (.setAutoCommit conn autocommit))))))
+
+(defn- with-index-write-lock
+ [pgvector f]
+ (with-index-lock pgvector "pg_advisory_lock" "pg_advisory_unlock" f))
+
(defn doc-id
"Content-addressed primary key for an index document.
`instructions` is intentionally not an input: editing it must not re-embed an entity's name/synonyms."
@@ -416,28 +457,19 @@
`ensure-tables!` (which may drop+rebuild on a model/format change) runs under the lock too, so a rebuild
can't pull the table out from under a concurrent node's in-flight run."
[pgvector resolve-model f]
- (with-open [^Connection conn (jdbc/get-connection pgvector)]
- ;; Per-batch commits, not one big transaction: the run tolerates partial failure across runs, so each
- ;; insert/delete must commit on its own. Some pools hand out autocommit-off connections (which would
- ;; silently roll the whole run back on close), so force it on and restore the prior setting before the
- ;; connection goes back to the pool. (ensure-tables! flips this to a transaction for its DDL.)
- (let [autocommit (.getAutoCommit conn)]
- (.setAutoCommit conn true)
- (try
- (jdbc/execute! conn [(format "SELECT pg_advisory_lock(%d)" reconcile-lock-id)])
- (try
- (let [embedding-model (resolve-model)
- status (index-table/ensure-tables! conn embedding-model)
- ;; :created (first build / healed manual drop) and :rebuilt (model/format change) both leave
- ;; an empty table, so a targeted caller must do a full repopulate rather than index one entity.
- emptied? (contains? #{:created :rebuilt} status)]
- (when emptied?
- (log/info "library entity index: vectors table is empty (" status "); repopulating"))
- (assoc (f conn embedding-model emptied?) :rebuilt? (= :rebuilt status)))
- (finally
- (jdbc/execute! conn [(format "SELECT pg_advisory_unlock(%d)" reconcile-lock-id)])))
- (finally
- (.setAutoCommit conn autocommit))))))
+ (with-index-write-lock
+ pgvector
+ (fn [conn]
+ ;; Per-batch commits, not one big transaction: the run tolerates partial failure across runs. The
+ ;; shared lock helper forces autocommit on; ensure-tables! temporarily wraps its DDL in a transaction.
+ (let [embedding-model (resolve-model)
+ status (index-table/ensure-tables! conn embedding-model)
+ ;; :created (first build / healed manual drop) and :rebuilt (model/format change) both leave
+ ;; an empty table, so a targeted caller must do a full repopulate rather than index one entity.
+ emptied? (contains? #{:created :rebuilt} status)]
+ (when emptied?
+ (log/info "library entity index: vectors table is empty (" status "); repopulating"))
+ (assoc (f conn embedding-model emptied?) :rebuilt? (= :rebuilt status))))))
(defn reconcile!
"Full reconcile of the pgvector `library_entity_index` with the appdb, blocking until it completes;
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
index ce43fba805a..5b355584d85 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
@@ -83,7 +83,7 @@
(do
(log/infof "Starting migration from version %d to %d."
db-version semantic.db.migration.impl/schema-version)
- (semantic.db.migration.impl/migrate-schema! tx opts)
+ (semantic.db.migration.impl/migrate-schema! tx (assoc opts :from-version db-version))
(write-successful-migration! index-metadata tx))
:else
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index 810e3f12b66..8f1b0957ae1 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -1,11 +1,13 @@
(ns metabase-enterprise.semantic-search.db.migration.impl
(:require
+ [clojure.string :as str]
[honey.sql :as sql]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.collections.curation :as collections.curation]
[metabase.config.core :as config]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.util :as u]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -15,7 +17,7 @@
(def schema-version
"Version to compare the [[metabase-enterprise.semantic-search.db.migration/db-version]] with. If this is higher,
schema migration will be performed."
- 2)
+ 3)
(def ^:private app-db-sentinel-tables
"Tables specific to a Metabase application database, chosen to avoid generic names (e.g. Liquibase's
@@ -91,14 +93,56 @@
{:drop-table [[[:raw (str (semantic.util/quote-ident schemaname) "."
(semantic.util/quote-ident tablename))]]]})))))
+(defn- quoted-table-name
+ [table-name]
+ (->> (semantic.util/qualified-table-parts table-name)
+ (remove nil?)
+ (map semantic.util/quote-ident)
+ (str/join ".")))
+
+(defn- add-embedding-space-metadata!
+ "Migration 3: preserve existing semantic indexes and add immutable embedding-space metadata."
+ [tx index-metadata]
+ (let [table-name (:metadata-table-name index-metadata)
+ control-table-name (:control-table-name index-metadata)
+ table (keyword table-name)]
+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN IF NOT EXISTS embedding_space_id TEXT"
+ (quoted-table-name table-name))])
+ ;; Legacy rows have no trustworthy revision provenance. Keep this nullable and let their backfilled
+ ;; embedding-space ID represent the old, explicitly unspecified revision.
+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN IF NOT EXISTS model_revision TEXT"
+ (quoted-table-name table-name))])
+ (doseq [{:keys [id provider model_name vector_dimensions]}
+ (jdbc/execute! tx
+ (sql/format {:select [:id :provider :model_name :vector_dimensions]
+ :from [table]})
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (let [space-id (:embedding-space-id
+ (embeddings.provider/legacy-resolved-model
+ {:provider provider :model-name model_name :vector-dimensions vector_dimensions}))]
+ (jdbc/execute! tx
+ (sql/format {:update table
+ :set {:embedding_space_id space-id}
+ :where [:= :id id]}))))
+ (jdbc/execute! tx [(format "ALTER TABLE %s ALTER COLUMN embedding_space_id SET NOT NULL"
+ (quoted-table-name table-name))])
+ (jdbc/execute! tx
+ (sql/format {:update (keyword control-table-name)
+ :set {:version (:version index-metadata)}
+ :where [:= :id 0]}
+ :quoted true))))
+
(defn migrate-schema!
"Migrate schema (control, metadata, gate, ...). Migration author is responsible for removing leftovers if necessary
and in general leaving schema in desired state."
- [tx {:keys [index-metadata] :as _opts}]
- ;; ideally index_table indexed are manipulated in dynamic schema part but for now it does not matter
- (drop-all-but-migration-table index-metadata tx)
- (semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)
- (semantic.index-metadata/ensure-control-row-exists! tx index-metadata))
+ [tx {:keys [index-metadata from-version] :as _opts}]
+ (if (= 2 from-version)
+ (add-embedding-space-metadata! tx index-metadata)
+ (do
+ ;; Older versions retain the existing reset migration. Only v2 has enough metadata to backfill safely.
+ (drop-all-but-migration-table index-metadata tx)
+ (semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)
+ (semantic.index-metadata/ensure-control-row-exists! tx index-metadata))))
(defn ensure-schema-compatibility!
"Apply non-destructive additions that existing databases at the current schema version also need."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index 509471f96f1..aecd4f33aef 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -1,5 +1,7 @@
(ns metabase-enterprise.semantic-search.embedding
(:require
+ [buddy.core.codecs :as buddy-codecs]
+ [buddy.core.hash :as buddy-hash]
[clj-http.client :as http]
[clojure.string :as str]
[diehard.circuit-breaker :as dh.cb]
@@ -8,6 +10,7 @@
[metabase-enterprise.semantic-search.settings :as semantic-settings]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.core :as analytics.core]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.llm.settings :as llm.settings]
[metabase.premium-features.core :as premium-features]
[metabase.tracing.core :as tracing]
@@ -46,14 +49,23 @@
(defn clean-provider-name
"Clean up a provider names for use in index names."
[provider-name]
- (str/replace provider-name #"[-:.]" "_"))
+ (-> provider-name
+ (str/replace #"[^A-Za-z0-9_]" "_")
+ (str/replace #"_{2,}" "_")
+ (str/replace #"^_+|_+$" "")))
(defn abbrev-provider-name
"Abbreviate long provider names for use in index names."
[provider-name]
(case provider-name
"ai-service" "ais"
- (clean-provider-name provider-name)))
+ "ollama" "ollama"
+ "openai" "openai"
+ "in-process" "inproc"
+ ;; Every plugin provider uses the same reserved form, including already-safe names. Otherwise an unsafe name's
+ ;; encoded output could itself be registered as a safe provider name and collide with the same physical index.
+ (str "plugin_" (clean-provider-name provider-name) "_"
+ (subs (buddy-codecs/bytes->hex (buddy-hash/sha1 provider-name)) 0 16))))
;;; Token Counting for OpenAI Models
@@ -133,23 +145,57 @@
(conj batches current-batch)
batches)))
-;;;; Provider SPI
-
-(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))
+;;;; Provider API
-(defmulti get-embedding
- "Returns a single embedding vector for the given text.
- `opts` (kwargs, honoured by the production providers; ollama ignores them):
+(defn- record-in-process-token-usage!
+ [{:keys [provider model-name]} texts {:keys [record-tokens? type]}]
+ (when (= "in-process" provider)
+ ;; The bundled tokenizer is intentionally private to the plugin artifact. Use the existing cl100k
+ ;; counter as an operational estimate so local inference participates in the same metrics/tracking.
+ (let [tokens (count-tokens-batch texts)]
+ (analytics/inc! :metabase-search/semantic-embedding-tokens
+ {:provider provider :model model-name}
+ tokens)
+ (when record-tokens?
+ (semantic.models.token-tracking/record-tokens model-name type tokens)))))
+
+(defn resolve-model
+ "Resolve a requested embedding model to its immutable vector-space descriptor."
+ [embedding-model]
+ (embeddings.provider/resolve-model embedding-model))
+
+(defn- in-process-token-model
+ [{:keys [provider model-name] :as embedding-model}]
+ (when (= "in-process" provider)
+ ;; Configured and persisted descriptors already carry the canonical name. Only the optional plugin default
+ ;; needs early resolution so token tracking has a stable label; the provider still owns embed-time resolution.
+ (if (nil? model-name)
+ (resolve-model embedding-model)
+ embedding-model)))
+
+(defn get-embedding
+ "Return one embedding vector for `text`.
+ `opts` (honoured by the production providers; ollama ignores them):
- `:record-tokens?` — write a token-tracking row for the call
- `:type` — what the embedding is for (`:query`/`:index`), recorded with the tokens
- `:snowplow?` — ai-service only, default true; synthetic callers (e.g. the health probe) pass
false so the call emits no token_usage event"
- {:arglists '([embedding-model text & opts])} dispatch-provider)
-
-(defmulti get-embeddings-batch
- "Returns a sequential collection of embedding vectors, in the same order as the input texts.
+ [embedding-model text & {:as opts}]
+ (let [token-model (in-process-token-model embedding-model)]
+ (u/prog1 (embeddings.provider/embed-text (or token-model embedding-model) text opts)
+ (when token-model
+ (record-in-process-token-usage! token-model [text] opts)))))
+
+(defn get-embeddings-batch
+ "Return one embedding vector per input text, in the same order.
Takes the same `opts` as [[get-embedding]], minus `:snowplow?` (batch callers are all organic)."
- {:arglists '([embedding-model texts & opts])} dispatch-provider)
+ [embedding-model texts & {:as opts}]
+ (let [token-model (in-process-token-model embedding-model)]
+ (u/prog1 (embeddings.provider/embed-texts (or token-model embedding-model) texts opts)
+ (when token-model
+ (record-in-process-token-usage! token-model texts opts)))))
+
+(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))
(defmulti embedder-circuit-endpoint
"Return the remote endpoint that identifies `embedding-model`'s circuit, or nil when its provider does not
@@ -158,9 +204,10 @@
(defmethod embedder-circuit-endpoint :default [_] nil)
-(defmulti pull-model
- "If a model needs to be downloaded (which is the case for ollama), downloads it."
- {:arglists '([embedding-model])} dispatch-provider)
+(defn pull-model
+ "Prepare a provider model eagerly when the provider supports it."
+ [embedding-model]
+ (embeddings.provider/prepare! embedding-model))
;;;; Embedding-service circuit breaker
;;;
@@ -170,6 +217,10 @@
;;; failing. State transitions surface the affected health checks immediately (see the listeners) rather
;;; than waiting for the daily job. Thresholds are fixed (the breaker is built once); the setting is a
;;; runtime kill switch checked per call.
+;;;
+;;; The breaker wraps the HTTP providers defined below rather than sitting behind the provider SPI, since
+;;; it keys circuits by remote endpoint and an in-process provider has none. If a second provider that
+;;; needs it turns up, move it into `metabase.embeddings.provider` instead of adding a second copy.
(def ^:private embedder-circuit-breaker-failure-threshold
"Consecutive embedding-service failures that trip the breaker open." 5)
@@ -385,11 +436,7 @@
(throw e))))
;; Ollama is not used in production. Token tracking is not implemented.
-(defmethod get-embedding "ollama" [{:keys [model-name vector-dimensions]} text & {:as _opts}]
- (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)
- :endpoint ollama-embeddings-endpoint))
-
-(defmethod get-embeddings-batch "ollama" [{:keys [model-name vector-dimensions]} texts & {:as _opts}]
+(defn- ollama-get-embeddings-batch [model-name vector-dimensions texts]
;; Ollama doesn't have a native batch API, so we fall back to individual calls. Each call goes through the
;; breaker on its own, so an outage mid-batch fast-fails the remaining texts instead of timing out on each.
(log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")
@@ -398,8 +445,6 @@
:endpoint ollama-embeddings-endpoint))
texts))
-(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))
-
;;;; OpenAI-compatible embedding service impl (shared by "ai-service" and "openai" providers)
(defn- supports-dimensions?
@@ -528,22 +573,8 @@
(defmethod embedder-circuit-endpoint "ai-service" [_]
(first (embedding-service-resolve-config!)))
-(defmethod get-embedding "ai-service"
- [{:keys [model-name vector-dimensions]} text & {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]
- (let [[endpoint api-key] (embedding-service-resolve-config!)]
- (first (openai-compatible-get-embeddings-batch
- {:provider "ai-service"
- :endpoint endpoint
- :api-key api-key
- :model-name model-name
- :vector-dimensions vector-dimensions
- :texts [text]
- :snowplow? snowplow?
- :record-tokens? record-tokens?
- :type type}))))
-
-(defmethod get-embeddings-batch "ai-service"
- [{:keys [model-name vector-dimensions]} texts & {:keys [record-tokens? type]}]
+(defn- ai-service-get-embeddings-batch
+ [{:keys [model-name vector-dimensions]} texts {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]
(let [[endpoint api-key] (embedding-service-resolve-config!)]
(openai-compatible-get-embeddings-batch
{:provider "ai-service"
@@ -552,13 +583,10 @@
:model-name model-name
:vector-dimensions vector-dimensions
:texts texts
- :snowplow? true
+ :snowplow? snowplow?
:record-tokens? record-tokens?
:type type})))
-(defmethod pull-model "ai-service" [_]
- (log/debug "ai-service provider does not require pulling a model"))
-
;;;; OpenAI provider
(defn- openai-resolve-config!
@@ -572,23 +600,8 @@
(defmethod embedder-circuit-endpoint "openai" [_]
(first (openai-resolve-config!)))
-(defmethod get-embedding "openai"
- [embedding-model text & {:keys [record-tokens? type]}]
- (let [[endpoint api-key] (openai-resolve-config!)]
- (first (openai-compatible-get-embeddings-batch
- {:provider "openai"
- :endpoint endpoint
- :api-key api-key
- :model-name (:model-name embedding-model)
- :vector-dimensions (:vector-dimensions embedding-model)
- :texts [text]
- :record-tokens? record-tokens?
- :extra-body (when (supports-dimensions? embedding-model)
- {:dimensions (:vector-dimensions embedding-model)})
- :type type}))))
-
-(defmethod get-embeddings-batch "openai"
- [embedding-model texts & {:keys [record-tokens? type]}]
+(defn- openai-get-embeddings-batch
+ [embedding-model texts {:keys [record-tokens? type]}]
(let [[endpoint api-key] (openai-resolve-config!)]
(openai-compatible-get-embeddings-batch
{:provider "openai"
@@ -602,18 +615,50 @@
{:dimensions (:vector-dimensions embedding-model)})
:type type})))
-(defmethod pull-model "openai" [_]
- (log/debug "OpenAI provider does not require pulling a model"))
+(defn- register-built-in-providers!
+ []
+ (let [spi-version embeddings.provider/embedding-spi-version
+ legacy embeddings.provider/legacy-resolved-model]
+ (embeddings.provider/register-provider!
+ "ollama"
+ {:embedding-spi-version spi-version
+ :readiness (constantly {:ready? true})
+ :resolve-model legacy
+ :embed-texts (fn [{:keys [model-name vector-dimensions]} texts _opts]
+ (ollama-get-embeddings-batch model-name vector-dimensions texts))
+ :prepare! (fn [{:keys [model-name]}]
+ (ollama-pull-model model-name))})
+ (embeddings.provider/register-provider!
+ "ai-service"
+ {:embedding-spi-version spi-version
+ :readiness (fn [_]
+ {:ready? (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))
+ (not-empty (llm.settings/ai-service-base-url))))})
+ :resolve-model legacy
+ :embed-texts ai-service-get-embeddings-batch})
+ (embeddings.provider/register-provider!
+ "openai"
+ {:embedding-spi-version spi-version
+ :readiness (fn [_]
+ {:ready? (boolean (not-empty (semantic-settings/openai-api-key)))})
+ :resolve-model legacy
+ :embed-texts openai-get-embeddings-batch})))
+
+(register-built-in-providers!)
;;;; Query prefixes for asymmetric retrieval models
(def ^:private model-family-query-prefixes
"Query prefixes for embedding-model families trained for asymmetric retrieval.
- These models expect search queries — but not the indexed documents — to carry a fixed prefix."
- ;; Patterns must be mutually exclusive: lookup scans entries in unspecified order.
+ These models expect search queries — but not the indexed documents — to carry a fixed prefix.
+ First match wins, so keep more specific patterns first."
+ ;; Arctic Embed moved to the short "query: " prefix at v2.0. Everything earlier expects the original
+ ;; longer instruction: v1 (xs, s, m, m-long, l) and v1.5 alike, including the bundled xs model.
;; Keep patterns narrow: a false positive is unfixable without a code change, since the
;; `ee-embedding-query-prefix` setting can only replace a matched prefix, never suppress it.
- {#"(?i)snowflake-arctic-embed" "query: "})
+ ;; Hence the leading anchor, so an unrelated model that merely contains the family name is left alone.
+ [[#"(?i)(?:^|/)snowflake-arctic-embed-\w+-v[2-9]" "query: "]
+ [#"(?i)(?:^|/)snowflake-arctic-embed(?:-|$)" "Represent this sentence for searching relevant passages: "]])
(defn- default-query-prefix
[model-name]
@@ -641,27 +686,10 @@
:model-name (semantic-settings/ee-embedding-model)
:vector-dimensions (semantic-settings/ee-embedding-model-dimensions)})
-(defmulti embedding-supported?
- "Whether `embedding-model`'s provider is *configured* to compute embeddings — the endpoint/credentials it
- needs are present. This is a config-presence check, not a liveness probe: a set URL whose service is down
- (or a stopped ollama) still reads as supported and surfaces at call time. Dispatches on provider,
- mirroring [[get-embedding]] and the config each provider's impl resolves; a new provider — including a
- future in-process embedder — adds a method. The `:default` is false, so an unrecognized provider gates
- callers off safely."
- {:arglists '([embedding-model])} dispatch-provider)
-
-(defmethod embedding-supported? :default [_] false)
-
-(defmethod embedding-supported? "ai-service" [_]
- (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))
- (not-empty (llm.settings/ai-service-base-url)))))
-
-(defmethod embedding-supported? "openai" [_]
- (boolean (not-empty (semantic-settings/openai-api-key))))
-
-;; ollama's endpoint is hardcoded (localhost:11434) with no setting to check, so config-presence is always
-;; true — consistent with ai-service/openai, which likewise check for a configured URL, not a live server.
-(defmethod embedding-supported? "ollama" [_] true)
+(defn embedding-supported?
+ "Whether the selected provider is installed and configured. This is not a remote liveness probe."
+ [embedding-model]
+ (embeddings.provider/ready? embedding-model))
(defn- calc-token-metrics
[texts]
@@ -704,7 +732,7 @@
(comment
;; Configuration:
- ;; MB_EE_EMBEDDING_PROVIDER: "ai-service" (default), "openai", or "ollama"
+ ;; MB_EE_EMBEDDING_PROVIDER: "ai-service" (default), "openai", "ollama", or "in-process"
;; MB_EE_EMBEDDING_MODEL: optional override (leave empty for provider defaults)
;; - OpenAI default: "text-embedding-3-small"
;; - Ollama default: "mxbai-embed-large"
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
index 14f43626796..f14a08791c2 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
@@ -85,8 +85,23 @@
index (:index state)
table-name (:table-name index)
stalled-at (-> state :metadata-row :indexer_stalled_at)
- model-mismatch? (not= (:embedding-model index) (semantic.env/get-configured-embedding-model))
- embedder-problem (when-not model-mismatch? (embedding-health/embedding-problem))
+ ;; The embedding space, not the settings, decides whether the stored vectors are still usable:
+ ;; a provider upgrade can resolve unchanged settings to a new space and strand the index.
+ ;; Resolution reaches the provider, so it can fail outright (plugin absent, platform
+ ;; unsupported) -- that is its own incident, not a mismatch.
+ configured (try
+ {:space-id (:embedding-space-id
+ (semantic.embedding/resolve-model
+ (semantic.env/get-configured-embedding-model)))}
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ {:error e}))
+ model-mismatch? (and (nil? (:error configured))
+ (not= (:embedding-space-id (:embedding-model index))
+ (:space-id configured)))
+ embedder-problem (when-not (or model-mismatch? (:error configured))
+ (embedding-health/embedding-problem))
problems (cond-> []
(not (active-index-queryable? pgvector table-name))
(conj "active index table not queryable")
@@ -97,6 +112,10 @@
(some? stalled-at)
(conj (str "indexer stalled since " stalled-at))
+ (some? (:error configured))
+ (conj (str "configured embedding model cannot be resolved: "
+ (ex-message (:error configured))))
+
model-mismatch?
(conj "active index embedding model does not match configured model")
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 112f22fef3c..ae43d578483 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -27,6 +27,7 @@
[toucan2.core :as t2])
(:import
[com.mchange.v2.c3p0 PooledDataSource]
+ [java.nio.charset StandardCharsets]
[java.time Instant LocalDate OffsetDateTime ZonedDateTime]
[java.util.concurrent ArrayBlockingQueue RejectedExecutionHandler RejectedExecutionException TimeUnit ThreadPoolExecutor]
[org.postgresql.util PGobject]))
@@ -271,7 +272,7 @@
Note: The index parameters will still be available in index_metadata"
[identifier]
- (if (<= (count identifier) 63)
+ (if (<= (alength (.getBytes ^String identifier StandardCharsets/UTF_8)) 63)
identifier
(str "index_" (buddy-codecs/bytes->hex (buddy-hash/sha1 identifier)))))
@@ -285,10 +286,13 @@
Table names produced here must stay recognizable by [[index-table-name?]]."
[embedding-model]
- (let [{:keys [model-name provider vector-dimensions]} embedding-model
+ (let [{:keys [model-name provider vector-dimensions embedding-space-id]} embedding-model
provider-name (embedding/abbrev-provider-name provider)
abbrev-model-name (embedding/abbrev-model-name model-name)
- ideal-table-name (str "index_" provider-name "_" abbrev-model-name "_" vector-dimensions)]
+ space-suffix (when embedding-space-id
+ (subs (buddy-codecs/bytes->hex (buddy-hash/sha1 embedding-space-id)) 0 12))
+ ideal-table-name (str "index_" provider-name "_" abbrev-model-name "_" vector-dimensions
+ (when space-suffix (str "_e" space-suffix)))]
(hash-identifier-if-exceeds-pg-limit ideal-table-name)))
(def ^:private index-table-name-pattern
@@ -296,8 +300,9 @@
suffix appended by [[metabase-enterprise.semantic-search.pgvector-api/fresh-index]]),
[[hash-identifier-if-exceeds-pg-limit]], and the legacy pre-BOT-337 naming era:
- index_<provider>_<model>_<dims> e.g. index_ais_text_3_sm_1536
- index_<provider>_<model>_<dims>_<digits> force-reset suffix ([[model-table-suffix]])
+ index_<provider>_<model>_<dims> legacy name
+ index_<provider>_<model>_<dims>_e<12-hex> immutable-space name
+ either modern name followed by _<digits> force-reset suffix ([[model-table-suffix]])
index_<40-hex-sha1> names exceeding the 63-byte pg identifier limit
index_table_<anything> legacy pre-BOT-337 naming (index_table_<provider>_<model>_<dims>)
@@ -306,7 +311,7 @@
gives the modern shapes their structure, while the index_table_ prefix — used exclusively by the
legacy era — claims anything under it. Deliberately does NOT match the control-plane tables
(index_metadata, index_control, index_gate): no trailing _<digits>, not 40-hex, not index_table_."
- #"\Aindex_(?:.+_\d+|[0-9a-f]{40}|table_.+)\z")
+ #"\Aindex_(?:.+_\d+(?:_e[0-9a-f]{12})?(?:_\d+)?|[0-9a-f]{40}|table_.+)\z")
(defn index-table-name?
"Does the bare (schema- and qualifier-stripped) table name look like a semantic-search index table?
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
index d68d7c3cf61..91bd6f8b379 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
@@ -2,7 +2,7 @@
"Multi-index management for pgvector semantic search.
Tracks multiple embedding model indexes through metadata tables, supporting model switching
- without data loss. Each embedding model (provider/name/dimensions) gets its own index table,
+ without data loss. Each immutable embedding space gets its own index table,
with a control table managing which index is currently active."
(:require
[honey.sql :as sql]
@@ -11,6 +11,7 @@
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.util :as u]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -30,7 +31,9 @@
[[:id :bigint [:primary-key] [:raw "GENERATED BY DEFAULT AS IDENTITY"]]
[:provider :text :not-null]
[:model_name :text :not-null]
+ [:model_revision :text :null]
[:vector_dimensions :int :not-null]
+ [:embedding_space_id :text :not-null]
[:table_name :text :not-null :unique]
[:index_version :int :not-null]
[:index_created_at :timestamp-with-time-zone :not-null]
@@ -70,14 +73,21 @@
[index index-metadata]
;; if created databases for namespacing in tests we could probably remove this whole idea.
(let [{:keys [index-table-qualifier]} index-metadata
+ ;; Re-apply the 63-byte hash: the qualifier can push an already-fit name over Postgres's limit,
+ ;; and a name silently truncated on CREATE would no longer match table-exists? lookups. The limit
+ ;; is per identifier, so hash only the table component -- a schema qualifier (app-db mode) is a
+ ;; separate short identifier that must survive, or the table lands outside its isolated schema.
qualify (fn [table-name]
- (format index-table-qualifier table-name))]
+ (let [[schema table] (semantic.util/qualified-table-parts
+ (format index-table-qualifier table-name))]
+ (cond->> (semantic.index/hash-identifier-if-exceeds-pg-limit table)
+ schema (str schema "."))))]
(-> index
(update :table-name qualify))))
(def default-index-metadata
"The index metadata configuration for a dedicated pgvector database (MB_PGVECTOR_DB_URL)."
- {:version "2"
+ {:version "3"
:metadata-table-name "index_metadata"
:control-table-name "index_control"
:gate-table-name "index_gate"
@@ -92,7 +102,7 @@
[[metabase-enterprise.semantic-search.util/table-name-part]]."
;; HoneySQL renders the dotted keywords as schema-qualified identifiers, quoted or not.
(let [schema semantic.db.datasource/app-db-schema]
- {:version "2"
+ {:version "3"
:schema schema
:metadata-table-name (str schema ".index_metadata")
:control-table-name (str schema ".index_control")
@@ -222,12 +232,17 @@
(defn- row->index [{:keys [provider
model_name
+ model_revision
vector_dimensions
+ embedding_space_id
table_name
index_version]}]
- {:embedding-model {:provider provider
- :model-name model_name
- :vector-dimensions vector_dimensions}
+ {:embedding-model (cond-> {:provider provider
+ :model-name model_name
+ :vector-dimensions vector_dimensions
+ :embedding-space-id embedding_space_id
+ :embedding-spi-version embeddings.provider/embedding-spi-version}
+ model_revision (assoc :model-revision model_revision))
:table-name table_name
:version index_version})
@@ -297,7 +312,8 @@
{:keys [provider
model-name
- vector-dimensions]}
+ vector-dimensions
+ embedding-space-id]}
embedding-model
model-rows-sql (-> {:select [:m.* [[:coalesce [:= :c.active_id :m.id] false] :is_active]]
@@ -306,7 +322,8 @@
:where [:and
[:= :provider provider]
[:= :model_name model-name]
- [:= :vector_dimensions vector-dimensions]]
+ [:= :vector_dimensions vector-dimensions]
+ [:= :embedding_space_id embedding-space-id]]
:order-by [[:index_created_at :desc]]}
(sql/format :quoted true))
model-rows (jdbc/execute! pgvector model-rows-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})
@@ -361,13 +378,17 @@
{:keys [provider
model-name
- vector-dimensions]}
+ model-revision
+ vector-dimensions
+ embedding-space-id]}
embedding-model
insert-sql (-> (sql.helpers/insert-into (keyword metadata-table-name))
(sql.helpers/values [{:provider provider
:model_name model-name
+ :model_revision model-revision
:vector_dimensions vector-dimensions
+ :embedding_space_id embedding-space-id
:table_name index-table-name
:index_version index-version
:index_created_at [:now]}])
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
index c80c7819e2b..661aecb2c58 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
@@ -11,6 +11,7 @@
[metabase-enterprise.semantic-search.db.connection :as semantic.db.connection]
[metabase-enterprise.semantic-search.db.migration :as semantic.db.migration]
[metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.gate :as semantic.gate]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
@@ -30,7 +31,6 @@
(def pgvector (or @semantic.db/data-source (semantic.db/init-db!)))
(def index-metadata semantic.index-metadata/default-index-metadata)
- (require '[metabase-enterprise.semantic-search.embedding :as semantic.embedding])
(def embedding-model (semantic.embedding/get-configured-model)))
(defn- fresh-index [index-metadata embedding-model & {:keys [force-reset?]}]
@@ -118,10 +118,11 @@
Designed to be called once at application startup (or in tests)."
[pgvector index-metadata embedding-model & {:as opts}]
- (semantic.db.connection/with-migrate-tx [tx pgvector]
- (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata
- :embedding-model embedding-model})
- (initialize-index! tx index-metadata embedding-model opts)))
+ (let [embedding-model (semantic.embedding/resolve-model embedding-model)]
+ (semantic.db.connection/with-migrate-tx [tx pgvector]
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata
+ :embedding-model embedding-model})
+ (initialize-index! tx index-metadata embedding-model opts))))
;; query/index-mgmt require an active index to be established first.
;; init-semantic-search! must be called on startup
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
index cf6970faf98..073fd6c0a91 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.semantic-search.settings
(:require
+ [clojure.string :as str]
[metabase.events.core :as events]
[metabase.llm.settings :as llm-settings]
[metabase.search.config :as search.config]
@@ -10,12 +11,8 @@
;; here, not there, so it's valid wherever the setter runs regardless of handler-namespace load order.
(events/derive! :event/semantic-search-hnsw-enabled :metabase/event)
-(def ^:private valid-embedding-providers
- "The set of valid embedding provider names."
- #{"ai-service" "openai" "ollama"})
-
(defsetting ee-embedding-provider
- (deferred-tru "The embedding provider to use (`openai`, `ollama`, or `ai-service`)")
+ (deferred-tru "The registered embedding provider to use")
:encryption :no
:visibility :settings-manager
:default "ai-service"
@@ -23,11 +20,12 @@
:export? false
:doc false
:setter (fn [new-value]
- (when (and new-value (not (contains? valid-embedding-providers new-value)))
- (throw (ex-info (str "Invalid embedding provider: " (pr-str new-value)
- ". Valid providers are: " (pr-str valid-embedding-providers))
- {:invalid-value new-value
- :valid-values valid-embedding-providers})))
+ ;; Provider plugins may be configured before they are loaded. Readiness reports an unregistered
+ ;; provider as unavailable, so validation here only needs to protect the registry key contract.
+ (when-not (or (nil? new-value)
+ (and (string? new-value) (not (str/blank? new-value))))
+ (throw (ex-info "Embedding provider must be a non-blank string."
+ {:invalid-value new-value})))
(setting/set-value-of-type! :string :ee-embedding-provider new-value)))
(defsetting ee-embedding-model
@@ -41,8 +39,9 @@
(defsetting ee-embedding-query-prefix
(deferred-tru
(str "Prefix prepended to search queries (but not indexed documents) before embedding them, as expected by "
- "asymmetric retrieval models such as the snowflake-arctic-embed family (`query: `). It is prepended "
- "verbatim, so include any trailing separator. Leave empty to use the default for the model family."))
+ "asymmetric retrieval models such as the snowflake-arctic-embed family. It is prepended verbatim, so "
+ "include any trailing separator. Leave empty to use the default for the model, which varies by "
+ "generation: Arctic Embed v2.0 expects `query: `, earlier versions expect a longer instruction."))
:encryption :no
:visibility :settings-manager
:default nil
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
index f72c3f0473b..5252fbe790e 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj
@@ -11,6 +11,7 @@
[metabase-enterprise.data-complexity-score.synonym-source :as synonym-source]
[metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score]
[metabase.app-db.core :as mdb]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.json :as json]))
@@ -387,6 +388,22 @@
(#'cli/run-cli {:source "appdb" :write-to-appdb false})
(is (false? @persisted?))))))
+(deftest ^:sequential run-cli-appdb-mode-registers-in-process-provider-test
+ (testing "appdb mode registers the in-process embedding provider before it scores"
+ ;; The standalone CLI skips `metabase.core.core/init!`, where the server registers the provider. Without
+ ;; this a synonym axis configured for `in-process` fails with `:provider-not-registered`.
+ (let [order (atom [])]
+ (mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])
+ embeddings.startup/ensure-in-process-provider! (fn [] (swap! order conj :register))
+ complexity/complexity-scores (fn [& _]
+ (swap! order conj :score)
+ {:meta {}})
+ synonym-source/complexity-scores-opts (constantly {})
+ metabot-scope/internal-metabot-scope (constantly {})
+ data-complexity-score/record-score! (fn [& _])]
+ (#'cli/run-cli {:source "appdb" :write-to-appdb false})
+ (is (= [:register :score] @order))))))
+
(deftest ^:parallel dir-digest-is-stable-and-content-sensitive-test
(testing "dir-digest produces the same value for the same content"
(let [d1 (#'representation/dir-digest representation-fixture-dir)
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index 494c0287436..4ecd37baa6b 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -23,6 +23,7 @@
[metabase.audit-app.core :as audit]
[metabase.collections.core :as collections]
[metabase.collections.test-utils :as collections.tu]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.task.core :as task]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -31,9 +32,18 @@
(def ^:private test-entity-ids (atom 0))
-(defmethod semantic-search/get-embeddings-batch ::dimension-translation-test
- [embedding-model _texts & _opts]
- [embedding-model])
+(def ^:private captured-translation-model
+ "Descriptor the provider layer receives, captured by the dimension-translation test." (atom nil))
+
+;; A capturing provider so the dimension-translation test can inspect the descriptor the client hands down.
+(embeddings.provider/register-provider!
+ "dimension-translation-test"
+ {:embedding-spi-version embeddings.provider/embedding-spi-version
+ :readiness (constantly {:ready? true})
+ :resolve-model embeddings.provider/legacy-resolved-model
+ :embed-texts (fn [model texts _opts]
+ (reset! captured-translation-model model)
+ (mapv (fn [_] (vec (repeat (:vector-dimensions model) 0.0))) texts))})
(defn- entity
"Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`
@@ -734,6 +744,24 @@
(is (fn? embedder)
"synonym-source returns a fresh provider-embedder for the descriptor")))))
+(deftest ^:sequential synonym-source-in-process-opts-pin-bundled-minilm-test
+ (testing "the in-process synonym provider selects the bundled MiniLM model, not Library retrieval's Arctic model"
+ (test-util/with-synonym-source [:provider "in-process"]
+ (let [{:keys [embedder embedding-model-meta text-variant]} (synonym-source/complexity-scores-opts)
+ bundled-minilm {:provider "in-process"
+ :model-name "sentence-transformers/all-MiniLM-L6-v2"
+ :model-dimensions 384}
+ captured (atom nil)]
+ (is (= bundled-minilm embedding-model-meta))
+ (is (= :names-split text-variant))
+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch
+ (fn [model texts & _opts]
+ (reset! captured model)
+ (repeat (count texts) [1.0]))]
+ (embedder [{:id 1 :name "orders" :kind :table}]))
+ (is (= bundled-minilm @captured)
+ "the embedder asks the provider SPI for the bundled MiniLM model")))))
+
(deftest ^:sequential provider-embedder-suppresses-token-tracking-test
(testing "provider-embedder always passes :record-tokens? false to get-embeddings-batch"
;; Complexity scoring isn't user-driven search traffic; the score itself is the analytics
@@ -749,11 +777,10 @@
(is (false? (:record-tokens? @captured)))))))
(deftest ^:parallel embeddings-client-translates-neutral-dimension-key-test
- (let [[translated]
- (embeddings/get-embeddings-batch
- {:provider ::dimension-translation-test, :model-name "fake", :model-dimensions 384}
- ["orders"])]
- (is (= 384 (:vector-dimensions translated)))))
+ (embeddings/get-embeddings-batch
+ {:provider "dimension-translation-test", :model-name "fake", :model-dimensions 384}
+ ["orders"])
+ (is (= 384 (:vector-dimensions @captured-translation-model))))
(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test
(testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"
diff --git a/enterprise/backend/test/metabase_enterprise/embeddings/client_test.clj b/enterprise/backend/test/metabase_enterprise/embeddings/client_test.clj
new file mode 100644
index 00000000000..a21091c6678
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/embeddings/client_test.clj
@@ -0,0 +1,33 @@
+(ns metabase-enterprise.embeddings.client-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.embeddings.client :as embeddings.client]
+ [metabase-enterprise.semantic-search.core :as semantic-search]))
+
+(set! *warn-on-reflection* true)
+
+(deftest legacy-model-dimensions-are-normalized-test
+ (let [captured (atom nil)]
+ (with-redefs [semantic-search/get-embeddings-batch
+ (fn [model texts & {:as opts}]
+ (reset! captured [model texts opts])
+ [[1.0 2.0]])]
+ (is (= [[1.0 2.0]]
+ (embeddings.client/get-embeddings-batch
+ {:provider "ai-service" :model-name "model" :model-dimensions 2}
+ ["text"]
+ :record-tokens? false)))
+ (is (= [{:provider "ai-service" :model-name "model" :vector-dimensions 2}
+ ["text"]
+ {:record-tokens? false}]
+ @captured)))))
+
+(deftest conflicting-dimension-keys-are-rejected-test
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"conflicting dimension values"
+ (embeddings.client/get-embeddings-batch
+ {:provider "ai-service"
+ :model-name "model"
+ :model-dimensions 2
+ :vector-dimensions 3}
+ ["text"]))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
index 6966c4046d1..da427d0695f 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
@@ -46,6 +46,85 @@
(is (> (:total_score (score {:distance 0.1 :doc_type "name"}))
(:total_score (score {:distance 0.5 :doc_type "name"})))))))
+(deftest search-uses-reconcile-lock-test
+ (let [calls (atom [])
+ lock-count (atom 0)]
+ (mt/with-premium-features #{:library-retrieval}
+ (with-redefs [entity-retrieval.core/available? (constantly true)
+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ reconcile/with-index-read-lock (fn [ds f]
+ (let [n (swap! lock-count inc)
+ conn (keyword (str "locked-connection-" n))]
+ (swap! calls conj [:lock ds])
+ (f conn)))
+ index-table/index-status (fn [conn _model]
+ (swap! calls conj [:compatible conn])
+ :compatible)
+ semantic.embedding/get-embedding (fn [_model text & _opts]
+ (swap! calls conj [:embed text])
+ [0.0 0.0 0.0 0.0])
+ jdbc/execute! (fn [conn & _args]
+ (swap! calls conj [:query conn])
+ [])]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10)))
+ (is (= [[:lock ::datasource]
+ [:compatible :locked-connection-1]
+ [:embed "the query"]
+ [:lock ::datasource]
+ [:compatible :locked-connection-2]
+ [:query :locked-connection-2]]
+ @calls)
+ "both preflight and query compatibility checks happen behind non-blocking reconcile locks")))))
+
+(deftest search-skips-embedding-for-an-incompatible-index-test
+ (mt/with-premium-features #{:library-retrieval}
+ (with-redefs [entity-retrieval.core/available? (constantly true)
+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ index-table/index-status (constantly :incompatible)
+ semantic.embedding/get-embedding (fn [& _]
+ (throw (ex-info "must not embed" {})))
+ reconcile/with-index-read-lock (fn [_ds f]
+ (f ::locked-connection))]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+
+(deftest search-degrades-when-reconcile-holds-the-lock-test
+ (mt/with-premium-features #{:library-retrieval}
+ (with-redefs [entity-retrieval.core/available? (constantly true)
+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ index-table/index-status (fn [& _]
+ (throw (ex-info "must not touch index metadata" {})))
+ semantic.embedding/get-embedding (fn [& _]
+ (throw (ex-info "must not embed" {})))
+ reconcile/with-index-read-lock (constantly nil)
+ jdbc/execute! (fn [& _]
+ (throw (ex-info "must not query" {})))]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+
+(deftest search-degrades-when-configured-model-cannot-be-resolved-test
+ (mt/with-premium-features #{:library-retrieval}
+ (with-redefs [entity-retrieval.core/available? (constantly true)
+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ semantic.embedding/resolve-model (fn [_]
+ (throw (ex-info "model changed"
+ {:type ::model-changed})))
+ index-table/index-status (fn [& _]
+ (throw (ex-info "must not inspect index" {})))
+ semantic.embedding/get-embedding (fn [& _]
+ (throw (ex-info "must not embed" {})))
+ reconcile/with-index-read-lock (fn [& _]
+ (throw (ex-info "must not lock" {})))
+ jdbc/execute! (fn [& _]
+ (throw (ex-info "must not query" {})))]
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+
(deftest dispatch-without-pgvector-test
(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"
;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:
@@ -302,15 +381,18 @@
(testing "search ranks library documents by cosine similarity, nearest first"
(when semantic.db.datasource/db-url
(mt/with-premium-features #{:library :library-retrieval}
- (let [suffix (System/nanoTime)
- ds (semantic.db.datasource/ensure-initialized-data-source!)
- q "the query"]
+ (let [suffix (System/nanoTime)
+ ds (semantic.db.datasource/ensure-initialized-data-source!)
+ q "the query"
+ prefixed-q "query: the query"
+ model (semantic.tu/resolved-mock-embedding-model
+ :model-name "Snowflake/snowflake-arctic-embed-l-v2.0")]
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
- (constantly semantic.tu/mock-embedding-model)]
+ (constantly model)]
(binding [index-table/*tables* {:vectors (str "library_entity_index_test_" suffix)
:meta (str "library_entity_index_meta_test_" suffix)}]
- ;; the two tables' name docs embed "near"/"far"; q ties "near" exactly and is orthogonal to "far".
- (semantic.tu/with-mock-embeddings {q [1.0 0.0 0.0 0.0]
+ ;; Arctic's prefixed query ties "near" exactly and is orthogonal to "far".
+ (semantic.tu/with-mock-embeddings {prefixed-q [1.0 0.0 0.0 0.0]
"near" [1.0 0.0 0.0 0.0]
"far" [0.0 1.0 0.0 0.0]}
(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}
@@ -321,7 +403,7 @@
:model/Table {far-id :id} {:db_id db-id :collection_id data-id :is_published true
:active true :name "far" :display_name "far"}]
(try
- (reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model))
+ (reconcile/reconcile! ds (constantly model))
(testing "the nearer table ranks first; each hit carries the entity ref + matched doc"
(is (=? [{:entity {:model "table" :id near-id} :doc_type "name" :doc_text "near"}
{:entity {:model "table" :id far-id}}]
@@ -456,8 +538,21 @@
(is (=? [{:model "table" :id table-id}]
(distinct (map :entity (entity-retrieval.core/search-unfiltered "orders" 10))))
"the query really runs, so the [] below is the degrade path and not a fallback")
- ;; a dim-1 query vector against the dim-4 index column -> pgvector SQLState 22000
- (is (= [] (entity-retrieval.core/search-unfiltered "q" 10))))
+ ;; Shrink the configured model to one dimension, the way an upgrade would, keeping it
+ ;; self-consistent so the provider resolves it instead of rejecting a changed space.
+ ;; The meta row still describes the dim-4 build, so the compatibility preflight would
+ ;; answer [] on its own; stub it aside to reach the case this test is about — a dim-1
+ ;; literal meeting the vector(4) column, which is pgvector SQLState 22000.
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly (semantic.tu/resolved-mock-embedding-model
+ :vector-dimensions 1))
+ index-table/index-status (constantly :compatible)]
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.entity-retrieval.core :warn]]
+ (is (= [] (entity-retrieval.core/search-unfiltered "q" 10)))
+ (is (=? [{:level :warn
+ :message #"(?s)library entity index incompatible with the query vector.*"}]
+ (messages))
+ "the query reached pgvector and degraded on 22000, not on an earlier preflight"))))
(finally
(jdbc/execute! ds [(str "DROP TABLE IF EXISTS \"" (index-table/vectors-table)
"\", \"" (index-table/meta-table) "\"")])))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
index a15415a005c..56ffee646fb 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
@@ -60,75 +60,74 @@
(is (= ["CREATE SCHEMA IF NOT EXISTS \"library_retrieval\""] @statements)
"nothing else creates it: semantic search provisions its own, and we run without semantic search")))
-(deftest legacy-tables-are-moved-not-rebuilt-test
+(deftest incompatible-legacy-tables-are-dropped-test
(let [statements (atom [])
- exists (atom #{"library_entity_index" "library_entity_index_meta"})]
- (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))
- #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)
- #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}
+ located (atom {"library_entity_index" {:schema-name "legacy.with.dot"
+ :table-name "library_entity_index"}
+ "library_entity_index_meta" {:schema-name "legacy.with.dot"
+ :table-name "library_entity_index_meta"}})]
+ (with-redefs-fn {#'index-table/legacy-table-in-search-path (fn [_ table] (get @located table))
+ #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)
+ #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}
(fn []
- (testing "an index built before the schema existed is moved, keeping its rows"
- (#'index-table/adopt-legacy-tables! ::tx)
- (is (= ["ALTER TABLE \"library_entity_index\" SET SCHEMA \"library_retrieval\""
- "ALTER TABLE \"library_entity_index_meta\" SET SCHEMA \"library_retrieval\""]
+ (testing "an index built before immutable space identity is discarded"
+ (#'index-table/drop-legacy-tables! ::tx)
+ (is (= ["DROP TABLE \"legacy.with.dot\".\"library_entity_index\""
+ "DROP TABLE \"legacy.with.dot\".\"library_entity_index_meta\""]
@statements)))
- (testing "and left alone once moved"
+ (testing "there is nothing to do after the legacy tables are gone"
(reset! statements [])
- (reset! exists #{"library_retrieval.library_entity_index"
- "library_retrieval.library_entity_index_meta"})
- (#'index-table/adopt-legacy-tables! ::tx)
+ (reset! located {})
+ (#'index-table/drop-legacy-tables! ::tx)
(is (empty? @statements)))
- (testing "a qualified table standing beside a leftover legacy one keeps its rows"
+ (testing "same-named app db tables are never dropped"
(reset! statements [])
- (reset! exists #{"library_entity_index"
- "library_entity_index_meta"
- "library_retrieval.library_entity_index"
- "library_retrieval.library_entity_index_meta"})
- (#'index-table/adopt-legacy-tables! ::tx)
- (is (empty? @statements)
- "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))
- (testing "an app db is never adopted from: nothing there can be an old index of ours"
- (reset! statements [])
- (reset! exists #{"library_entity_index" "library_entity_index_meta"})
+ (reset! located {"library_entity_index" {:schema-name "public"
+ :table-name "library_entity_index"}
+ "library_entity_index_meta" {:schema-name "public"
+ :table-name "library_entity_index_meta"}})
(with-redefs [semantic.db.datasource/db-url nil]
- (#'index-table/adopt-legacy-tables! ::tx))
+ (#'index-table/drop-legacy-tables! ::tx))
(is (empty? @statements)
- "an application table on the search path answers to the bare name, and must not be moved"))))))
+ "application tables with the legacy names do not belong to the library index"))))))
-(deftest ^:synchronized legacy-tables-are-adopted-in-a-dedicated-store-test
- (testing "a real SET SCHEMA carries the rows an upgrading dedicated store already embedded"
+(deftest ^:synchronized incompatible-legacy-tables-are-replaced-in-a-dedicated-store-test
+ (testing "old vectors without immutable space identity are replaced with current empty tables"
(when (semantic.db.datasource/dedicated-url-configured?)
- ;; its own database: adoption only runs under the real table names, which a shared store may hold
+ ;; its own database: cleanup only runs under the real table names, which a shared store may hold
(semantic.tu/with-test-db! {:dbname "library_retrieval_adoption_test" :mode :blank :cleanup :both}
(let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)]
(jdbc/execute! pgvector ["CREATE EXTENSION IF NOT EXISTS vector"])
- ;; both bare names the index carried before it had a schema of its own, with a meta row for the
- ;; model it was built against -- an upgrade the reconciler should find nothing to redo
- (jdbc/execute! pgvector ["CREATE TABLE library_entity_index
- (doc_id text primary key, entity_type text, entity_local_id bigint,
- doc_type text, doc_text text, doc_embedding vector(4))"])
- (jdbc/execute! pgvector ["INSERT INTO library_entity_index
- VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])
- (jdbc/execute! pgvector ["CREATE TABLE library_entity_index_meta
- (id smallint primary key, provider text, model_name text,
- vector_dimensions int, schema_version int,
- updated_at timestamptz, reconciled_at timestamptz)"])
- (jdbc/execute! pgvector ["INSERT INTO library_entity_index_meta
- VALUES (1, ?, ?, ?, ?, now(), now())"
- (:provider semantic.tu/mock-embedding-model)
- (:model-name semantic.tu/mock-embedding-model)
- (:vector-dimensions semantic.tu/mock-embedding-model)
- index-table/schema-version])
- (is (= :ok (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model))
- "the adopted meta row still describes the configured model, so nothing is rebuilt")
- (is (= ["legacy-doc"]
- (map :doc_id (jdbc/execute! pgvector
- [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
- "moved, not recreated empty -- otherwise the whole library re-embeds on upgrade")
- (is (empty? (jdbc/execute! pgvector [(str "SELECT tablename FROM pg_tables WHERE schemaname = 'public'"
- " AND tablename LIKE 'library_entity_index%'")]))
- "moved, not copied -- a leftover is a stale duplicate of the whole index"))))))
+ (jdbc/with-transaction [tx pgvector]
+ (jdbc/execute! tx ["CREATE SCHEMA \"legacy.retrieval\""])
+ (jdbc/execute! tx ["SET LOCAL search_path TO \"legacy.retrieval\", public"])
+ ;; both bare names and the metadata shape the index carried before immutable space identity
+ (jdbc/execute! tx ["CREATE TABLE library_entity_index
+ (doc_id text primary key, entity_type text, entity_local_id bigint,
+ doc_type text, doc_text text, doc_embedding vector(4))"])
+ (jdbc/execute! tx ["INSERT INTO library_entity_index
+ VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])
+ (jdbc/execute! tx ["CREATE TABLE library_entity_index_meta
+ (id smallint primary key, provider text, model_name text,
+ vector_dimensions int, schema_version int,
+ updated_at timestamptz, reconciled_at timestamptz)"])
+ (jdbc/execute! tx ["INSERT INTO library_entity_index_meta
+ VALUES (1, ?, ?, ?, ?, now(), now())"
+ (:provider semantic.tu/mock-embedding-model)
+ (:model-name semantic.tu/mock-embedding-model)
+ (:vector-dimensions semantic.tu/mock-embedding-model)
+ index-table/schema-version])
+ (#'index-table/drop-legacy-tables! tx))
+ (is (= :created (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model)))
+ (is (empty? (jdbc/execute! pgvector
+ [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+ "unknown legacy vectors are not relabeled as the current embedding space")
+ (is (empty? (jdbc/execute! pgvector
+ [(str "SELECT schemaname, tablename FROM pg_tables"
+ " WHERE schemaname <> 'library_retrieval'"
+ " AND tablename LIKE 'library_entity_index%'")]))
+ "the incompatible tables do not remain in their custom search-path schema"))))))
(deftest reconcile-watermark-precedes-appdb-read-test
(let [events (atom [])]
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
index e1693850d91..c4dd5ca27b6 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
@@ -13,7 +13,9 @@
[metabase.test.fixtures :as fixtures]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as jdbc.rs]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.sql Connection)))
(set! *warn-on-reflection* true)
@@ -93,6 +95,30 @@
(index-table/vectors-table) ", "
(index-table/meta-table))])))))))
+(deftest ^:sequential with-index-read-lock-contention-integration-test
+ (when semantic.db.datasource/db-url
+ (let [ds (semantic.db.datasource/ensure-initialized-data-source!)
+ lock-id (var-get #'reconcile/reconcile-lock-id)
+ calls (atom 0)
+ callback (fn [_]
+ (swap! calls inc)
+ ::called)]
+ (with-open [^Connection writer (jdbc/get-connection ds)]
+ (let [acquired? (:acquired
+ (jdbc/execute-one!
+ writer
+ [(format "SELECT pg_try_advisory_lock(%d) AS acquired" lock-id)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))]
+ (is acquired? "the exclusive test lock was acquired without blocking")
+ (when acquired?
+ (try
+ (is (nil? (reconcile/with-index-read-lock ds callback)))
+ (is (zero? @calls) "the read callback is skipped while reconcile owns the lock")
+ (finally
+ (jdbc/execute! writer [(format "SELECT pg_advisory_unlock(%d)" lock-id)])))
+ (is (= ::called (reconcile/with-index-read-lock ds callback)))
+ (is (= 1 @calls) "the callback runs after the exclusive lock is released")))))))
+
(defn- index-rows [ds]
(jdbc/execute! ds
[(format "SELECT doc_id, entity_type, entity_local_id, doc_type, doc_text FROM \"%s\""
@@ -201,7 +227,7 @@
(is (seq (docs-for ds "table" table-id)))
(is (some? (reconciled-at! ds)) "a converged reconcile stamps reconciled_at"))
(testing "a model-identity change drops the vectors table and the next run re-embeds everything"
- (let [new-model (assoc model :model-name "model-v2")]
+ (let [new-model (semantic.tu/resolved-mock-embedding-model :model-name "model-v2")]
(is (= :rebuilt (index-table/ensure-tables! ds new-model)))
(is (= [] (index-rows ds)))
(is (nil? (reconciled-at! ds))
@@ -216,6 +242,72 @@
(testing "the rebuild heals the meta row, so it doesn't recur on the next sync"
(is (= :ok (index-table/ensure-tables! ds new-model)))))))))))
+(deftest ^:sequential embedding-space-change-rebuilds-test
+ (with-isolated-index [ds]
+ (let [model semantic.tu/mock-embedding-model
+ changed-space (update model :embedding-space-id str "-changed")]
+ (is (= :created (index-table/ensure-tables! ds model)))
+ (jdbc/execute! ds [(format (str "INSERT INTO \"%s\" "
+ "(doc_id, entity_type, entity_local_id, doc_type, doc_text, doc_embedding) "
+ "VALUES ('sentinel', 'table', 1, 'name', 'sentinel', '[0,0,0,0]')")
+ (index-table/vectors-table))])
+ (testing "the same provider/name/dimensions with a different immutable space is incompatible"
+ (is (= :incompatible (index-table/index-status ds changed-space)))
+ (is (= :rebuilt (index-table/ensure-tables! ds changed-space)))
+ (is (empty? (index-rows ds))))
+ (testing "the rebuilt identity is stable"
+ (is (= :compatible (index-table/index-status ds changed-space)))
+ (is (= :ok (index-table/ensure-tables! ds changed-space)))))))
+
+(deftest ^:sequential version-1-meta-upgrade-rebuilds-test
+ (with-isolated-index [ds]
+ (let [model semantic.tu/mock-embedding-model]
+ (jdbc/execute! ds [(format (str "CREATE TABLE \"%s\" ("
+ "id smallint PRIMARY KEY, provider text NOT NULL, model_name text NOT NULL, "
+ "vector_dimensions int NOT NULL, schema_version int NOT NULL, "
+ "updated_at timestamptz NOT NULL)")
+ (index-table/meta-table))])
+ (jdbc/execute! ds [(format (str "INSERT INTO \"%s\" "
+ "(id, provider, model_name, vector_dimensions, schema_version, updated_at) "
+ "VALUES (1, 'mock', 'model', 4, 1, NOW())")
+ (index-table/meta-table))])
+ (jdbc/execute! ds [(format "CREATE TABLE \"%s\" (sentinel int)" (index-table/vectors-table))])
+ (jdbc/execute! ds [(format "INSERT INTO \"%s\" VALUES (1)" (index-table/vectors-table))])
+ (testing "a legacy table is rebuilt once rather than relabeled as the resolved space"
+ (is (= :rebuilt (index-table/ensure-tables! ds model)))
+ (is (empty? (index-rows ds))))
+ (let [meta-row (jdbc/execute-one! ds
+ [(format (str "SELECT embedding_space_id, schema_version "
+ "FROM \"%s\" WHERE id = 1")
+ (index-table/meta-table))]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ column-row (jdbc/execute-one! ds
+ ["SELECT is_nullable FROM information_schema.columns WHERE table_name = ? AND column_name = 'embedding_space_id'"
+ (index-table/meta-table)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (is (= {:embedding_space_id (:embedding-space-id model)
+ :schema_version index-table/schema-version}
+ meta-row))
+ (is (= "NO" (:is_nullable column-row))))
+ (let [execute! jdbc/execute!
+ statements (atom [])]
+ (with-redefs [jdbc/execute! (fn [connectable sql-params & opts]
+ (swap! statements conj (first sql-params))
+ (apply execute! connectable sql-params opts))]
+ (is (= :ok (index-table/ensure-tables! ds model))))
+ (is (not-any? #(re-find #"ALTER COLUMN embedding_space_id SET NOT NULL" (str %)) @statements)
+ "steady-state reconcile does not reacquire an ACCESS EXCLUSIVE lock"))
+ (testing "a manually nullable metadata column is healed once"
+ (jdbc/execute! ds [(format "ALTER TABLE \"%s\" ALTER COLUMN embedding_space_id DROP NOT NULL"
+ (index-table/meta-table))])
+ (is (= :ok (index-table/ensure-tables! ds model)))
+ (is (= "NO" (:is_nullable
+ (jdbc/execute-one!
+ ds
+ ["SELECT is_nullable FROM information_schema.columns WHERE table_name = ? AND column_name = 'embedding_space_id'"
+ (index-table/meta-table)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))))))
+
(deftest ^:sequential measures-and-segments-indexed-and-hydrated-test
(testing "measures/segments on a published library table are indexed and hydrate with parent-table context"
(mt/with-premium-features #{:library :library-retrieval}
@@ -427,7 +519,7 @@
(is (seq (docs-for ds "table" b-id)))
;; a targeted reconcile of A under a new model identity forces ensure-tables! to rebuild (empty);
;; it must repopulate B too, not leave it missing until the periodic backstop.
- (let [new-model (assoc model :model-name "model-v2")]
+ (let [new-model (semantic.tu/resolved-mock-embedding-model :model-name "model-v2")]
(is (:rebuilt? (reconcile/reconcile-entity! ds (constantly new-model) "table" a-id)))
(is (seq (docs-for ds "table" a-id)) "A repopulated after the rebuild")
(is (seq (docs-for ds "table" b-id)) "B repopulated too (not dropped by the rebuild)")))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj
index 9a81ec992f1..e6a0c155597 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj
@@ -10,13 +10,17 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase-enterprise.semantic-search.core :as semantic.core]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase.app-db.core :as mdb]
+ [metabase.embeddings.provider :as embeddings.provider]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[next.jdbc :as jdbc]
@@ -67,6 +71,15 @@
[]
(Boolean/parseBoolean (System/getenv "MB_APPDB_PGVECTOR_MODE_TEST")))
+(defn- in-process-embedder-opted-in?
+ []
+ (Boolean/parseBoolean (System/getenv "MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST")))
+
+(def ^:private in-process-embedding-model
+ {:provider "in-process"
+ :model-name "Snowflake/snowflake-arctic-embed-xs"
+ :vector-dimensions 384})
+
(deftest ^:synchronized appdb-pgvector-round-trip-test
(cond
(not (opted-in?))
@@ -118,7 +131,7 @@
(is (contains? tables "index_metadata"))
(is (contains? tables "index_control"))
(is (contains? tables "index_gate"))
- (is (contains? tables "index_mock_model_4"))
+ (is (contains? tables (semantic.index/model-table-name semantic.tu/mock-embedding-model)))
(is (some #(str/starts-with? % "dlq_") tables))))
(testing "search round-trips through the app-db pool"
;; results are reconstructed from legacy_input, so the mock card comes back as model+id
@@ -133,3 +146,54 @@
(is (= public-before (tables-in-schema app-db "public")))))
(finally
(jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"])))))))))
+
+(deftest ^:synchronized appdb-pgvector-in-process-embedder-round-trip-test
+ (cond
+ (not (in-process-embedder-opted-in?))
+ (testing "real in-process inference requires its dedicated CI opt-in — skipping"
+ (is true))
+
+ (not (opted-in?))
+ (testing "the in-process round trip requires the app-db pgvector opt-in"
+ (is false "MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST requires MB_APPDB_PGVECTOR_MODE_TEST"))
+
+ (not (appdb-can-host-pgvector?))
+ (testing "opted in, but the app db can't host pgvector"
+ (is false "the in-process round trip expects a pgvector-capable Postgres app db"))
+
+ (schema-exists? (mdb/data-source) "semantic_search")
+ (testing "opted in, but a semantic_search schema already exists"
+ (is false "the appdb-mode job should start from a fresh app db with no semantic_search schema"))
+
+ :else
+ (mt/with-premium-features #{:semantic-search}
+ (embeddings.startup/ensure-in-process-provider!)
+ (is (embeddings.provider/registered? "in-process"))
+ (is (semantic.embedding/embedding-supported? in-process-embedding-model))
+ (with-redefs [semantic.db.datasource/db-url nil
+ semantic.db.datasource/data-source (atom nil)
+ semantic.db.datasource/app-db-pgvector-support (atom nil)
+ semantic.db.datasource/probe-cooldown-timer (atom nil)
+ semantic.db.datasource/logged-pgvector-absent? (atom false)
+ semantic.embedding/get-configured-model (constantly in-process-embedding-model)]
+ (let [app-db (mdb/data-source)]
+ (try
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ documents (mapv #(assoc % :archived false) (semantic.tu/mock-documents))]
+ (is (semantic.core/supported?) "the semantic search engine accepts the registered provider")
+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata
+ (semantic.env/get-configured-embedding-model))
+ (semantic.pgvector-api/index-documents! pgvector index-metadata documents)
+ (testing "real in-process embeddings flow through indexing and semantic querying"
+ (is (= {:model "card" :id 123}
+ (-> (semantic.tu/with-weights {:semantic-distance 1}
+ (mt/with-test-user :crowberto
+ (semantic.pgvector-api/query pgvector index-metadata
+ {:search-string "puppy"
+ :vector-search-strategy :brute-force})))
+ :results
+ first
+ (select-keys [:model :id]))))))
+ (finally
+ (jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"]))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index c1b44b1b897..7e308388f15 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -9,11 +9,13 @@
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.indexer :as semantic.indexer]
+ [metabase-enterprise.semantic-search.models.token-tracking :as semantic.models.token-tracking]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.snowplow-test :as snowplow-test]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.llm.settings :as llm.settings]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
@@ -52,7 +54,17 @@
(is (= {:provider "openai"
:model-name "text-embedding-3-small"
:vector-dimensions 1536}
- (embedding/get-configured-model))))))
+ (embedding/get-configured-model))))
+ (mt/with-temporary-setting-values [ee-embedding-provider "in-process"
+ ee-embedding-model "Snowflake/snowflake-arctic-embed-l-v2.0"
+ ee-embedding-model-dimensions 1024]
+ (is (= {:provider "in-process"
+ :model-name "Snowflake/snowflake-arctic-embed-l-v2.0"
+ :vector-dimensions 1024}
+ (embedding/get-configured-model))))
+ (testing "plugin-defined provider names do not require a core allowlist change"
+ (mt/with-temporary-setting-values [ee-embedding-provider "openrouter-plugin"]
+ (is (= "openrouter-plugin" (:provider (embedding/get-configured-model))))))))
(deftest test-model-dimensions-with-settings
(testing "model-dimensions uses setting defaults when override is nil"
@@ -62,18 +74,39 @@
(mt/with-temporary-setting-values [ee-embedding-model-dimensions 768]
(is (= 768 (:vector-dimensions (embedding/get-configured-model)))))))
+(deftest embedding-provider-setting-validation-test
+ (doseq [invalid-value [42 [] {} " "]]
+ (testing (str "rejects " (pr-str invalid-value))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"must be a non-blank string"
+ (semantic.settings/ee-embedding-provider! invalid-value))))))
+
(deftest prefix-search-query-test
- (let [arctic {:provider "ai-service" :model-name "Snowflake/snowflake-arctic-embed-l-v2.0" :vector-dimensions 1024}
- openai {:provider "openai" :model-name "text-embedding-3-small" :vector-dimensions 1536}]
- (mt/with-temporary-setting-values [ee-embedding-query-prefix nil]
- (testing "arctic-family models get the `query: ` prefix by default"
- (is (= "query: hello" (embedding/prefix-search-query arctic "hello"))))
- (testing "models without a known family default are left unprefixed"
- (is (= "hello" (embedding/prefix-search-query openai "hello")))))
- (testing "the setting overrides the model-family default, verbatim"
- (mt/with-temporary-setting-values [ee-embedding-query-prefix "search_query: "]
- (is (= "search_query: hello" (embedding/prefix-search-query arctic "hello")))
- (is (= "search_query: hello" (embedding/prefix-search-query openai "hello")))))))
+ (let [cqe "Represent this sentence for searching relevant passages: "
+ ->model (fn [model-name] {:provider "ai-service" :model-name model-name :vector-dimensions 1024})
+ prefix-all (fn [model-names]
+ (into {}
+ (map (juxt identity #(embedding/prefix-search-query (->model %) "hello")))
+ model-names))]
+ (testing "the default prefix follows the Arctic Embed generation, which shortened it at v2.0"
+ (let [expected {"Snowflake/snowflake-arctic-embed-xs" (str cqe "hello")
+ "Snowflake/snowflake-arctic-embed-l" (str cqe "hello")
+ "Snowflake/snowflake-arctic-embed-m-long" (str cqe "hello")
+ "Snowflake/snowflake-arctic-embed-m-v1.5" (str cqe "hello")
+ "Snowflake/snowflake-arctic-embed-l-v2.0" "query: hello"
+ "Snowflake/snowflake-arctic-embed-m-v2.0" "query: hello"
+ "text-embedding-3-small" "hello"
+ ;; merely containing the family name is not enough: the prefix would be unsuppressable
+ "acme/my-snowflake-arctic-embed-clone" "hello"}]
+ (mt/with-temporary-setting-values [ee-embedding-query-prefix nil]
+ (is (= expected (prefix-all (keys expected)))))))
+ (testing "the setting overrides every model-family default, verbatim"
+ (let [model-names ["Snowflake/snowflake-arctic-embed-xs"
+ "Snowflake/snowflake-arctic-embed-l-v2.0"
+ "text-embedding-3-small"]]
+ (mt/with-temporary-setting-values [ee-embedding-query-prefix "search_query: "]
+ (is (= (zipmap model-names (repeat "search_query: hello"))
+ (prefix-all model-names))))))))
(deftest test-openai-provider-validation
(testing "OpenAIProvider throws when API key not configured"
@@ -97,6 +130,50 @@
(is (zero? (#'embedding/count-tokens "")))
(is (nil? (#'embedding/count-tokens nil)))))
+(deftest in-process-token-usage-test
+ (let [model {:provider "in-process" :model-name "local-model" :vector-dimensions 4}
+ unnamed-model (dissoc model :model-name)
+ analytics-calls (atom [])
+ tracking-calls (atom [])
+ resolution-calls (atom [])]
+ (with-redefs [embeddings.provider/resolve-model (fn [requested]
+ (swap! resolution-calls conj requested)
+ (assoc requested :model-name "local-model"))
+ embeddings.provider/embed-text (fn [_ text _] [text])
+ embeddings.provider/embed-texts (fn [_ texts _] (mapv vector texts))
+ analytics/inc! (fn [metric labels value]
+ (swap! analytics-calls conj [metric labels value]))
+ semantic.models.token-tracking/record-tokens (fn [& args]
+ (swap! tracking-calls conj args))]
+ (embedding/get-embedding model "Hello world" :type :query :record-tokens? true)
+ (embedding/get-embeddings-batch model ["Hello world" "again"] :type :index :record-tokens? true)
+ (testing "local calls report approximate token metrics and persistent usage"
+ (is (= [[:metabase-search/semantic-embedding-tokens
+ {:provider "in-process" :model "local-model"}
+ 2]
+ [:metabase-search/semantic-embedding-tokens
+ {:provider "in-process" :model "local-model"}
+ 3]]
+ @analytics-calls))
+ (is (= [["local-model" :query 2]
+ ["local-model" :index 3]]
+ @tracking-calls)))
+ (testing "named local models rely on the provider's embed-time resolution"
+ (is (empty? @resolution-calls)))
+ (testing "the adapter neither resolves nor counts other providers"
+ (embedding/get-embedding (assoc model :provider "openai") "Hello world")
+ (is (empty? @resolution-calls))
+ (is (= 2 (count @analytics-calls)))
+ (is (= 2 (count @tracking-calls))))
+ (testing "an unnamed local request resolves once to obtain its token label"
+ (embedding/get-embedding unnamed-model "Hello world" :type :query :record-tokens? true)
+ (is (= [unnamed-model] @resolution-calls))
+ (is (= [:metabase-search/semantic-embedding-tokens
+ {:provider "in-process" :model "local-model"}
+ 2]
+ (last @analytics-calls)))
+ (is (= ["local-model" :query 2] (last @tracking-calls)))))))
+
(deftest test-batching-logic
(testing "create-batches handles empty input"
(is (empty? (#'embedding/create-batches 10 count [])))
@@ -331,6 +408,32 @@
"tag" "embedding_generation"}}]
events))))))))
+(deftest test-embedding-service-snowplow-suppression
+ (testing "ai-service fires no token_usage event when the caller passes :snowplow? false"
+ ;; The health and circuit-recovery probes embed a synthetic string; counting those as organic usage
+ ;; would inflate the token_usage series by however often the probes run.
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://mock-embedding-service"
+ ee-embedding-service-api-key "mock-key"]
+ (let [mock-response {:data [{:object "embedding"
+ :embedding (encode-floats-to-base64 [1.0 2.0 3.0])
+ :index 0}]
+ :model "test-model"
+ :usage {:prompt_tokens 5
+ :total_tokens 5}}]
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [http/post (fn [_url & _opts]
+ {:status 200
+ :headers {"Content-Type" "application/json"}
+ :body (json/encode mock-response)})]
+ (embedding/get-embedding {:provider "ai-service"
+ :model-name "test-model"
+ :vector-dimensions 3}
+ "health check"
+ {:type :query, :record-tokens? false, :snowplow? false}))
+ (let [events (->> (snowplow-test/pop-event-data-and-user-id!)
+ (filter #(= "embedding_generation" (get-in % [:data "tag"]))))]
+ (is (empty? events))))))))
+
(deftest ^:sequential token-tracking-write-test
(mt/with-premium-features #{:semantic-search}
(when (string? (not-empty (:mb-pgvector-db-url env/env)))
@@ -401,4 +504,17 @@
(is (false? (embedding/embedding-supported? {:provider "openai"})))))
(testing "ollama is always supported; an unrecognized provider is not (:default)"
(is (true? (embedding/embedding-supported? {:provider "ollama"})))
- (is (false? (embedding/embedding-supported? {:provider "no-embedder"})))))
+ (is (false? (embedding/embedding-supported? {:provider "no-embedder"}))))
+ (testing "in-process can be configured before its plugin is installed, but is not ready without it"
+ (is (false? (embedding/embedding-supported? {:provider "in-process"})))))
+
+(deftest resolve-model-test
+ (let [requested {:provider "openai"
+ :model-name "text-embedding-3-small"
+ :vector-dimensions 1536}
+ resolved (embedding/resolve-model requested)]
+ (is (= requested (select-keys resolved (keys requested))))
+ (is (= 1 (:embedding-spi-version resolved)))
+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}" (:embedding-space-id resolved)))
+ (testing "transport credentials are not part of vector-space identity"
+ (is (= resolved (embedding/resolve-model (assoc requested :api-key "do-not-persist")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
index b77feeae078..07ecbf07ee7 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj
@@ -255,8 +255,10 @@
(deftest flush-watermark!-test
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
- model1 {:provider "foo" :model-name "m1" :vector-dimensions 42}
- model2 {:provider "foo" :model-name "m2" :vector-dimensions 42}
+ model1 (semantic.tu/resolved-mock-embedding-model
+ :provider "foo" :model-name "m1" :vector-dimensions 42)
+ model2 (semantic.tu/resolved-mock-embedding-model
+ :provider "foo" :model-name "m2" :vector-dimensions 42)
index1 (semantic.index-metadata/qualify-index (semantic.index/default-index model1) index-metadata)
index2 (semantic.index-metadata/qualify-index (semantic.index/default-index model2) index-metadata)
watermark {:last-poll (ts "2025-01-01T13:00:00Z")
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
index 8b0d0b8bb6a..760b0547cee 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
@@ -26,8 +26,23 @@
(def ^:private embedding-model
{:provider "ai-service", :model-name "test-model", :vector-dimensions 3})
+(def ^:private space-id
+ "The embedding space the active index was built in. Health compares this, not the settings keys."
+ "emb:v1:sha256:test")
+
+(def ^:private other-space-id
+ "A space the same settings could resolve to after a provider upgrade."
+ "emb:v1:sha256:old")
+
+(def ^:private active-index-embedding-model
+ "The active index's :embedding-model, as produced by index-metadata/row->index: it carries bookkeeping
+ keys (:embedding-space-id, :embedding-spi-version) that get-configured-embedding-model never returns."
+ (assoc embedding-model
+ :embedding-space-id space-id
+ :embedding-spi-version 1))
+
(def ^:private active-state
- {:index {:table-name "index_test_table", :embedding-model embedding-model}
+ {:index {:table-name "index_test_table", :embedding-model active-index-embedding-model}
:metadata-row {:id 1
:indexer_stalled_at nil
:repair_orphan_count nil
@@ -42,6 +57,7 @@
semantic.env/get-pgvector-datasource! (constantly ::pgvector)
semantic.env/get-index-metadata (constantly ::index-metadata)
semantic.env/get-configured-embedding-model (constantly embedding-model)
+ semantic.embedding/resolve-model (constantly active-index-embedding-model)
semantic.settings/semantic-search-vector-strategy (constantly :brute-force)
embedding-health/request-circuit-recovery! (constantly nil)]
(thunk)))
@@ -132,14 +148,22 @@
embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
(is (=? {:health 0 :message #".*required HNSW index is missing.*"}
(semantic.health/index-health-check)))))
- (testing "an active index for a different embedding model degrades without probing the wrong model"
+ (testing "an index built in another embedding space degrades without probing the wrong model"
(mt/with-dynamic-fn-redefs
[semantic.index-metadata/get-active-index-state
- (constantly (assoc-in active-state [:index :embedding-model :model-name] "old-model"))
+ (constantly (assoc-in active-state [:index :embedding-model :embedding-space-id] other-space-id))
semantic.health/active-index-queryable? (constantly true)
embedding-health/embedding-problem #(throw (ex-info "must not probe configured model" {}))]
(is (=? {:health 0 :message #".*active index embedding model does not match configured model.*"}
(semantic.health/index-health-check)))))
+ (testing "a configured model the provider cannot resolve degrades, and is not read as a mismatch"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/resolve-model (fn [_] (throw (ex-info "no provider" {})))
+ embedding-health/embedding-problem #(throw (ex-info "must not probe" {}))]
+ (is (=? {:health 0 :message #".*cannot be resolved: no provider.*"}
+ (semantic.health/index-health-check)))))
(testing "a non-closed breaker (open or half-open) still probes (so a recovered-but-idle embedder is
detectable) and degrades identically in both states"
(let [probed? (atom false)]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj
index d6ef17f91f8..178c5970a0c 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj
@@ -10,6 +10,13 @@
(def ^:private test-long-model-name "some-really-really-really-really-really-long-model-name-that-will-exceed-the-limit")
(def ^:private test-vector-dimensions 1024)
+(deftest ^:parallel hash-identifier-uses-postgres-byte-limit-test
+ (let [identifier (str "index_" (apply str (repeat 30 "é")))]
+ (is (<= (count identifier) 63) "the identifier fits when incorrectly counted as characters")
+ (is (re-matches #"index_[0-9a-f]{40}"
+ (semantic.index/hash-identifier-if-exceeds-pg-limit identifier))
+ "UTF-8 identifiers over PostgreSQL's 63-byte limit are hashed")))
+
;; Postgres truncates identifier names to 63 bytes (+ 1 byte for a terminating NULL). Ensure that names do not exceed the limit
(deftest ^:parallel index-embedding-name-length-test
(mt/with-premium-features #{:semantic-search}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj
index 74bab719e48..0c8a2f280a5 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj
@@ -5,6 +5,7 @@
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.test :as mt]
[metabase.util :as u])
(:import
@@ -23,7 +24,19 @@
(is (= {:table-name "bar"} (sut {:table-name "bar"} {:index-table-qualifier "%s"})))
(testing "qualifier applies to derived index names"
(is (= "foo_bar_embed_hnsw_idx" (-> (sut {:table-name "bar"} {:index-table-qualifier "foo_%s"})
- semantic.index/hnsw-index-name)))))))
+ semantic.index/hnsw-index-name)))))
+ (testing "hashes over-long identifiers within Postgres's 63-byte limit"
+ (let [long-table (apply str "index_" (repeat 60 "x"))
+ fits-table (apply str "index_" (repeat 42 "x"))]
+ (testing "an unqualified over-long name is hashed whole"
+ (is (= (semantic.index/hash-identifier-if-exceeds-pg-limit long-table)
+ (:table-name (sut {:table-name long-table} {:index-table-qualifier "%s"})))))
+ (testing "a schema qualifier is preserved; only the table component is hashed"
+ (is (= (str "semantic_search." (semantic.index/hash-identifier-if-exceeds-pg-limit long-table))
+ (:table-name (sut {:table-name long-table} {:index-table-qualifier "semantic_search.%s"})))))
+ (testing "a schema-qualified name over 63 bytes whose table component fits is left intact"
+ (is (= (str "semantic_search." fits-table)
+ (:table-name (sut {:table-name fits-table} {:index-table-qualifier "semantic_search.%s"})))))))))
(deftest create-tables-if-not-exists!-test
(let [pgvector (semantic.env/get-pgvector-datasource!)
@@ -116,6 +129,22 @@
:metadata-row (first (semantic.tu/get-metadata-rows pgvector index-metadata))}
(sut pgvector index-metadata))))))))
+(deftest revisioned-active-index-state-round-trip-test
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.tu/unique-index-metadata)
+ embedding-model (semantic.tu/resolved-mock-embedding-model :model-revision "revision-1")
+ index (-> (semantic.index/default-index embedding-model)
+ (semantic.index-metadata/qualify-index index-metadata))]
+ (with-open [_ (open-tables! pgvector index-metadata)]
+ (semantic.index-metadata/ensure-control-row-exists! pgvector index-metadata)
+ (let [index-id (semantic.index-metadata/record-new-index-table! pgvector index-metadata index)]
+ (semantic.index-metadata/activate-index! pgvector index-metadata index-id)
+ (let [{:keys [index metadata-row]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)
+ reloaded-model (:embedding-model index)]
+ (is (= embedding-model reloaded-model))
+ (is (= "revision-1" (:model_revision metadata-row)))
+ (is (= 4 (count (embeddings.provider/embed-text reloaded-model "round-trip")))))))))
+
(defn- default-index [embedding-model index-metadata]
(mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix semantic.tu/mock-table-suffix]
(-> (semantic.index/default-index embedding-model)
@@ -137,8 +166,7 @@
(deftest find-compatible-index!-test
(let [pgvector (semantic.env/get-pgvector-datasource!)
embedding-model1 semantic.tu/mock-embedding-model
- embedding-model2 (assoc semantic.tu/mock-embedding-model
- :model-name "mock2")
+ embedding-model2 (semantic.tu/resolved-mock-embedding-model :model-name "mock2")
sut semantic.index-metadata/find-compatible-index!
;; warning: the setup-scenario can currently only set up a happy path
;; other variables include:
@@ -190,6 +218,16 @@
(testing "no metadata"
(is (nil? (sut' model))))))))))))
+(deftest embedding-space-id-is-part-of-index-compatibility-test
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.tu/unique-index-metadata)
+ model semantic.tu/mock-embedding-model]
+ (with-open [_ (open-tables! pgvector index-metadata)]
+ (setup-scenario! pgvector index-metadata {:active model})
+ (is (some? (semantic.index-metadata/find-compatible-index! pgvector index-metadata model)))
+ (is (nil? (semantic.index-metadata/find-compatible-index!
+ pgvector index-metadata (assoc model :embedding-space-id "replacement-space")))))))
+
(deftest create-new-index-spec-test
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
@@ -217,7 +255,9 @@
(is (=? [{:id index-id
:provider (:provider embedding-model)
:model_name (:model-name embedding-model)
+ :model_revision nil
:vector_dimensions (:vector-dimensions embedding-model)
+ :embedding_space_id (:embedding-space-id embedding-model)
:table_name (:table-name index)
:index_version (:version index)}]
(semantic.tu/get-metadata-rows pgvector index-metadata)))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index dc9431f801a..098d71c1d3b 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -179,6 +179,8 @@
;; force-reset suffix, as pgvector-api/fresh-index builds it
(str (semantic.index/model-table-name semantic.tu/mock-embedding-model) "_" (semantic.index/model-table-suffix))
"index_ollama_mxbai_lg_1024_1234567"
+ "index_ollama_mxbai_lg_1024_e0123456789ab"
+ "index_ollama_mxbai_lg_1024_e0123456789ab_1234567"
;; hashed shape for names exceeding the pg identifier limit
(semantic.index/hash-identifier-if-exceeds-pg-limit (apply str "index_" (repeat 60 "x")))
(str "index_" (apply str (repeat 40 "a")))
@@ -203,6 +205,32 @@
(str "index_" (apply str (repeat 39 "a")))
(str "index_" (apply str (repeat 41 "a"))))))
+(deftest ^:parallel model-table-name-embedding-space-test
+ (let [model semantic.tu/mock-embedding-model]
+ (is (not= (semantic.index/model-table-name model)
+ (semantic.index/model-table-name (assoc model :embedding-space-id "another-space"))))
+ (is (= (str "index_" (semantic.embedding/abbrev-provider-name "mock") "_model_4")
+ (semantic.index/model-table-name (dissoc model :embedding-space-id))))
+ (testing "plugin provider identifiers occupy a disjoint, collision-resistant namespace"
+ (let [unsafe-name "vendor/foo"
+ encoded-name (semantic.embedding/abbrev-provider-name unsafe-name)
+ provider-names [unsafe-name "vendor-foo" "vendor foo" encoded-name]
+ table-names (mapv #(semantic.index/model-table-name (assoc model :provider %)) provider-names)]
+ (is (= (count provider-names) (count (distinct table-names))))
+ (is (not= encoded-name (semantic.embedding/abbrev-provider-name encoded-name))
+ "an encoded unsafe name must not collide when registered verbatim as another provider")
+ (doseq [[built-in alias] [["ai-service" "ais"]
+ ["in-process" "inproc"]
+ ["ollama" "plugin_ollama"]
+ ["openai" "plugin_openai"]]]
+ (is (not= (semantic.index/model-table-name (assoc model :provider built-in))
+ (semantic.index/model-table-name (assoc model :provider alias)))
+ (str "built-in alias must not collide with plugin provider " (pr-str alias))))
+ (doseq [table-name table-names]
+ (let [table-keyword (keyword table-name)]
+ (is (nil? (namespace table-keyword)))
+ (is (semantic.index/index-table-name? table-name))))))))
+
(defn- expected-index-defs
"The index-DDL contract of [[semantic.index/create-index-table-if-not-exists!]] for `index`.
Maps the name of each index it must create to a pattern its pg_indexes indexdef must match."
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
index 147ebf61df6..fa83987398c 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
@@ -15,6 +15,7 @@
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.collections.models.collection :as collection]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.log :as log]
@@ -58,6 +59,48 @@
(is (m/find-first (comp #{"Migration already performed, skipping."} :message)
(:messages <>)))))))))
+(deftest version-2-embedding-space-migration-preserves-index-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ table-name "index_legacy_preserved_4"
+ model {:provider "openai" :model-name "legacy-model" :vector-dimensions 4}
+ space-id (:embedding-space-id (embeddings.provider/legacy-resolved-model model))]
+ (jdbc/execute! pgvector ["CREATE TABLE migration (version bigint PRIMARY KEY, migrated_at timestamp DEFAULT NOW(), status varchar(32))"])
+ (jdbc/execute! pgvector ["INSERT INTO migration (version, status) VALUES (2, 'success')"])
+ (jdbc/execute! pgvector [(str "CREATE TABLE index_metadata ("
+ "id bigint PRIMARY KEY, provider text NOT NULL, model_name text NOT NULL, "
+ "vector_dimensions int NOT NULL, table_name text NOT NULL UNIQUE, "
+ "index_version int NOT NULL, index_created_at timestamptz NOT NULL)")])
+ (jdbc/execute! pgvector ["CREATE TABLE index_control (id bigint PRIMARY KEY, version text NOT NULL, active_id int, active_updated_at timestamptz)"])
+ (jdbc/execute! pgvector [(format "CREATE TABLE %s (id text PRIMARY KEY)" table-name)])
+ (jdbc/execute! pgvector [(str "INSERT INTO index_metadata "
+ "(id, provider, model_name, vector_dimensions, table_name, index_version, index_created_at) "
+ "VALUES (7, 'openai', 'legacy-model', 4, ?, 5, NOW())")
+ table-name])
+ (jdbc/execute! pgvector ["INSERT INTO index_control (id, version, active_id) VALUES (0, '2', 7)"])
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate!
+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))
+ (let [control-row (jdbc/execute-one! pgvector ["SELECT active_id, version FROM index_control"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ metadata-row (jdbc/execute-one! pgvector ["SELECT id, embedding_space_id, model_revision FROM index_metadata"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ column-rows (jdbc/execute! pgvector
+ ["SELECT column_name, is_nullable FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'index_metadata' AND column_name IN ('embedding_space_id', 'model_revision')"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ nullable-by-column (into {} (map (juxt :column_name :is_nullable)) column-rows)
+ active-model (get-in (semantic.index-metadata/get-active-index-state
+ pgvector semantic.index-metadata/default-index-metadata)
+ [:index :embedding-model])]
+ (testing "the physical index and active pointer survive"
+ (is (semantic.util/table-exists? pgvector table-name))
+ (is (= {:active_id 7 :version "3"} control-row)))
+ (testing "the existing metadata row is backfilled in place"
+ (is (= {:id 7 :embedding_space_id space-id :model_revision nil} metadata-row))
+ (is (= {"embedding_space_id" "NO" "model_revision" "YES"} nullable-by-column))
+ (is (not (contains? active-model :model-revision)))))))))
+
(deftest schema-scoped-migration-drop-test
(mt/with-premium-features #{:semantic-search}
(semantic.tu/with-test-db-defaults!
@@ -243,7 +286,9 @@
:indexer_last_seen
:indexer_last_seen_hash
:indexer_last_seen_id
+ :embedding_space_id
:model_name
+ :model_revision
:provider
:table_name
:vector_dimensions]))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
index dbc4e623eb5..7e4bcf34292 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
@@ -30,7 +30,7 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
model1 semantic.tu/mock-embedding-model
- model2 (assoc semantic.tu/mock-embedding-model :model-name "embed-harder")
+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embed-harder")
sut* semantic.pgvector-api/init-semantic-search!
cleanup (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))
sut #(semantic.tu/closeable (apply sut* %&) cleanup)]
@@ -179,7 +179,7 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
model1 semantic.tu/mock-embedding-model
- model2 (assoc semantic.tu/mock-embedding-model :model-name "embedagain")
+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embedagain")
sut semantic.pgvector-api/index-documents!]
(test-not-initialized sut pgvector index-metadata model1)
(with-open [index-ref (open-semantic-search! pgvector index-metadata model1)]
@@ -207,7 +207,7 @@
dash "dashboard"
card "card"
model1 semantic.tu/mock-embedding-model
- model2 (assoc semantic.tu/mock-embedding-model :model-name "embedagain")
+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embedagain")
sut semantic.pgvector-api/delete-documents!
documents (semantic.tu/mock-documents)
{card-ids "card", dash-ids "dashboard"} (u/group-by :model :id documents)]
@@ -246,7 +246,7 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
model1 semantic.tu/mock-embedding-model
- model2 (assoc semantic.tu/mock-embedding-model :model-name "judge-embedd")
+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "judge-embedd")
remove-scores (fn [rows] (mapv #(dissoc % :score :all-scores) rows)) ; scores have time-sensitives components
sut* semantic.pgvector-api/query
sut #(remove-scores (:results (mt/as-admin (apply sut* %&)))) ; see notes below about perms
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
index 51ffcd41e43..766f42a5bcd 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj
@@ -37,9 +37,12 @@
[table-name {:keys [provider model-name vector-dimensions]}]
{:table-name table-name
:version 1
- :embedding-model {:provider provider
+ ;; Resolve the model so it carries an :embedding-space-id -- record-new-index-table! persists it into a
+ ;; NOT NULL column.
+ :embedding-model (semantic.tu/resolved-mock-embedding-model
+ :provider provider
:model-name model-name
- :vector-dimensions vector-dimensions}})
+ :vector-dimensions vector-dimensions)})
(defn- insert-metadata-with-timestamps!
"Insert an index metadata row using record-new-index-table! and then update timestamps for testing."
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index f5e8629e22c..9639ec66757 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -17,6 +17,7 @@
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.embeddings.provider :as embeddings.provider]
[metabase.search.config :as search.config]
[metabase.search.ingestion :as search.ingestion]
[metabase.test :as mt]
@@ -278,10 +279,18 @@
;;;; mock provider
+(defn resolved-mock-embedding-model
+ "Build a self-consistent resolved descriptor for a mock model variant."
+ [& {:as overrides}]
+ (-> (merge {:provider "mock"
+ :model-name "model"
+ :vector-dimensions 4}
+ overrides)
+ embeddings.provider/legacy-resolved-model
+ (assoc :embedding-spi-version embeddings.provider/embedding-spi-version)))
+
(def mock-embedding-model
- {:provider "mock"
- :model-name "model"
- :vector-dimensions 4})
+ (resolved-mock-embedding-model))
(def mock-index-metadata
"An index metadata to qualify and isolate mock indexes"
@@ -306,15 +315,16 @@
(def mock-index
"A mock index for testing low level indexing functions.
Coincides with what the index-metadata system would create for the mock-embedding-model."
- (with-redefs [semantic.index/model-table-suffix mock-table-suffix]
+ (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix mock-table-suffix]
(-> (semantic.index/default-index mock-embedding-model)
(semantic.index-metadata/qualify-index mock-index-metadata))))
-;; NOTE: opts are currently unused in following mock implementations
-(defmethod semantic.embedding/get-embedding "mock" [_ text & {:as _opts}] (get-mock-embedding text))
-(defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts & {:as _opts}] (get-mock-embeddings-batch texts))
-(defmethod semantic.embedding/pull-model "mock" [_])
-(defmethod semantic.embedding/embedding-supported? "mock" [_] true)
+(embeddings.provider/register-provider!
+ "mock"
+ {:embedding-spi-version embeddings.provider/embedding-spi-version
+ :readiness (constantly {:ready? true})
+ :resolve-model embeddings.provider/legacy-resolved-model
+ :embed-texts (fn [_ texts _opts] (get-mock-embeddings-batch texts))})
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
(defn query-index [search-context]
diff --git a/modules/embedder/README.md b/modules/embedder/README.md
new file mode 100644
index 00000000000..4a3105d5e3a
--- /dev/null
+++ b/modules/embedder/README.md
@@ -0,0 +1,46 @@
+# In-process embedding plugin
+
+This module builds `metabase-embedder-plugin.jar`, an optional runtime plugin containing two pinned models:
+
+- `Snowflake/snowflake-arctic-embed-xs` (384 dimensions) — semantic search and Library retrieval
+- `sentence-transformers/all-MiniLM-L6-v2` (384 dimensions) — the data complexity score's synonym axis
+
+Both models are deliberately small (~23 MB per architecture) so CI can build the artifact and run real inference
+against it on every change without caching multi-GB downloads. They exercise the full packaging, tokenizer, and DJL
+runtime machinery. A release build bundling a larger retrieval model (such as `snowflake-arctic-embed-l-v2.0`) will
+be added when the plugin is published for customers or offline use such as evals.
+
+Build it with `./bin/build-embedder-plugin.sh`, then place the jar in Metabase's plugin directory. The plugin is
+discovered from the jar manifest at startup; each model's DJL runtime is loaded lazily on first inference, and only
+the models a consumer actually asks for are ever loaded.
+
+Consumers select the provider through their own settings. Semantic search and Library retrieval must also override
+the model and dimensions because their defaults describe the larger Arctic Embed model used by the AI service:
+
+```text
+MB_EE_EMBEDDING_PROVIDER=in-process
+MB_EE_EMBEDDING_MODEL=Snowflake/snowflake-arctic-embed-xs
+MB_EE_EMBEDDING_MODEL_DIMENSIONS=384
+```
+
+The data complexity score's default model and dimensions already match the bundled MiniLM model, so it only needs:
+
+```text
+MB_DATA_COMPLEXITY_SCORING_SYNONYM_EMBEDDING_PROVIDER=in-process
+```
+
+A model the plugin does not bundle, or configured dimensions that do not match it, fails readiness rather than
+falling back to another model.
+
+The artifact contains pinned ARM64 and x86-64/AVX2 ONNX exports. The supported runtime combinations are glibc 2.34
+or newer on Linux, on either architecture, and Apple Silicon macOS. Intel macOS is unsupported because the tokenizer
+dependency does not ship an x86-64 macOS native library. The runtime is not musl-compatible, so it cannot run in
+Metabase's default Alpine image; use a glibc-based image such as the repository's Ubuntu image when installing this
+plugin. Other operating systems, libc implementations, and architectures fail readiness checks before model loading.
+
+The exact embedding-space identity includes the architecture-specific export digest, so every node sharing
+semantic-search or library-retrieval indexes must use the same architecture. Mixed ARM64/x86-64 clusters fail closed
+rather than querying vectors produced by a different export.
+
+Custom model sources are intentionally not supported in this first version: the bundled catalog is the only place a
+model can come from.
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index 2e6abc43268..ca292b8fe89 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -16,6 +16,7 @@
[metabase.driver.mysql]
[metabase.driver.postgres]
[metabase.embedding.settings :as embed.settings]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.events.core :as events]
[metabase.initialization-status.core :as init-status]
[metabase.llm.startup :as llm.startup]
@@ -158,12 +159,6 @@
(catch Exception e
(log/warnf "Failed to register signal handler for SIG%s: %s" signal-name (ex-message e)))))))
-(def embedder-plugin-name
- "Manifest `info.name` of the in-process embedder. Pinned against the manifest by
- `metabase.embeddings.embedder-plugin-name-test`; a rename here without one there silently disables
- the plugin, since `plugins/registered?` looks it up by this exact string."
- "Metabase In-Process Embedder")
-
(defn- init!*
"General application initialization function which should be run once at application startup."
[]
@@ -184,13 +179,7 @@
(tracing/init!)
;; load any plugins as needed
(plugins/load-plugins!)
- ;; The in-process embedder registers its provider during plugin initialization. Its model runtime remains
- ;; lazy, so activating the lightweight registration namespace here does not load DJL or a model at startup.
- (when (plugins/registered? embedder-plugin-name)
- (try
- (plugins/load-plugin! embedder-plugin-name)
- (catch Exception e
- (log/warnf "Unable to activate the in-process embedder plugin: %s" (ex-message e)))))
+ (embeddings.startup/ensure-in-process-provider!)
(init-status/set-progress! 0.3)
(setting/validate-settings-formatting!)
;; startup database. validates connection & runs any necessary migrations
diff --git a/src/metabase/embeddings/startup.clj b/src/metabase/embeddings/startup.clj
new file mode 100644
index 00000000000..3440768f3aa
--- /dev/null
+++ b/src/metabase/embeddings/startup.clj
@@ -0,0 +1,31 @@
+(ns metabase.embeddings.startup
+ "Startup-time registration of the in-process embedding provider.
+
+ The provider registers itself while its plugin initializes, so every entry point that might embed has to run
+ this. The server does it during init; standalone modes that skip init have to do it themselves, or embedding
+ fails with `:provider-not-registered`."
+ (:require
+ [metabase.plugins.core :as plugins]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def embedder-plugin-name
+ "Manifest `info.name` of the in-process embedder.
+ Pinned against the manifest by `metabase.embeddings.embedder-plugin-name-test`; a rename here without
+ one there silently disables the plugin, since `plugins/registered?` looks it up by this exact string."
+ "Metabase In-Process Embedder")
+
+(defn ensure-in-process-provider!
+ "Register the in-process embedding provider when its plugin is installed.
+
+ Plugin loading is idempotent, so callers that have already loaded plugins pay nothing extra. Initialization only
+ registers the provider: DJL and the model itself stay lazy, so a server that never embeds loads neither. The
+ plugin ships separately from the uberjar, so its absence is the normal case and is not worth a warning."
+ []
+ (plugins/load-plugins!)
+ (when (plugins/registered? embedder-plugin-name)
+ (try
+ (plugins/load-plugin! embedder-plugin-name)
+ (catch Exception e
+ (log/warnf "Unable to activate the in-process embedder plugin: %s" (ex-message e))))))
diff --git a/test/metabase/embeddings/embedder_plugin_artifact_test.clj b/test/metabase/embeddings/embedder_plugin_artifact_test.clj
index 3ccbf87a56b..ca9b937ca2a 100644
--- a/test/metabase/embeddings/embedder_plugin_artifact_test.clj
+++ b/test/metabase/embeddings/embedder_plugin_artifact_test.clj
@@ -3,8 +3,8 @@
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase.core.core :as core]
[metabase.embeddings.provider :as embeddings.provider]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.plugins.impl :as plugins]
[metabase.plugins.initialize :as plugin-initialize])
(:import
@@ -12,11 +12,14 @@
(set! *warn-on-reflection* true)
-(def ^:private requested-model
+(def ^:private retrieval-model
+ "What semantic search and Library retrieval ask for."
{:provider "in-process"
:model-name "Snowflake/snowflake-arctic-embed-xs"})
-(def ^:private requested-minilm-model
+(def ^:private complexity-model
+ "What the data-complexity score's synonym axis asks for. Mean-pooled — unlike the cls-pooled retrieval model —
+ so it exercises a different translator configuration."
{:provider "in-process"
:model-name "sentence-transformers/all-MiniLM-L6-v2"})
@@ -65,6 +68,61 @@
(java.util.Collections/singletonList Proxy/NO_PROXY))
(connectFailed [_uri _socket-address _exception])))
+(defn- check-platform-support!
+ "Which platforms the bundled natives claim to support, driven through the catalog's private probes."
+ []
+ (let [libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)
+ os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
+ arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)
+ detect-libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'detect-linux-libc)
+ detect-libc (var-get detect-libc-var)
+ glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)
+ supported-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'supported-glibc-version?)
+ supported-version? (var-get supported-version-var)]
+ (is (= :unknown (detect-libc false nil)) "unreadable process maps fail closed")
+ (is (= :musl (detect-libc true nil)))
+ (is (= :musl (detect-libc false "/lib/ld-musl-x86_64.so.1")))
+ (is (= :glibc (detect-libc false "/usr/lib64/ld-2.28.so")))
+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2"))
+ "the Ubuntu 24.04 loader layout is recognized")
+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/libc.so.6")))
+ (is (false? (supported-version? "2.33")))
+ (is (true? (supported-version? "2.34")))
+ (is (true? (supported-version? "2.39")))
+ (is (false? (supported-version? nil)))
+ (with-redefs-fn {libc-var (constantly :glibc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.34")}
+ #(is (true? (:ready? (embeddings.provider/readiness retrieval-model)))))
+ (with-redefs-fn {libc-var (constantly :unknown)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.39")}
+ #(is (true? (:ready? (embeddings.provider/readiness retrieval-model)))
+ "the authoritative glibc API probe survives an inconclusive process map"))
+ (with-redefs-fn {libc-var (constantly :glibc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.33")}
+ #(do
+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc-version}
+ (embeddings.provider/readiness retrieval-model)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc-version"
+ (embeddings.provider/resolve-model retrieval-model)))))
+ (doseq [[unsupported-libc probed-glibc-version] [[:musl "2.39"]
+ [:unknown nil]]]
+ (with-redefs-fn {libc-var (constantly unsupported-libc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly probed-glibc-version)}
+ #(do
+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc}
+ (embeddings.provider/readiness retrieval-model)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc"
+ (embeddings.provider/resolve-model retrieval-model))))))
+ (with-redefs-fn {os-var (constantly "mac os x")
+ arch-var (constantly "avx2")}
+ #(is (= {:provider "in-process" :ready? false :reason :unsupported-platform}
+ (embeddings.provider/readiness retrieval-model))
+ "the tokenizer artifact has no Intel macOS native library"))))
+
(defn- check-missing-architecture!
"A catalog entry with no export for this architecture must fail rather than hash a nil sha256 into a
well-formed but meaningless embedding-space id."
@@ -86,12 +144,12 @@
(is true))
(let [pre-plugin-thread
(isolated-thread-call
- #(hash-map :readiness (embeddings.provider/readiness requested-model)
- :resolved (embeddings.provider/resolve-model requested-model)))]
+ #(hash-map :readiness (embeddings.provider/readiness retrieval-model)
+ :resolved (embeddings.provider/resolve-model retrieval-model)))]
(plugins/load-plugins!)
;; Non-driver plugins are registered eagerly but initialized only when a feature selects them. The provider
;; registration lives in the plugin init namespace, so activate this artifact explicitly before exercising it.
- (plugin-initialize/load-plugin! core/embedder-plugin-name)
+ (plugin-initialize/load-plugin! embeddings.startup/embedder-plugin-name)
(testing "the implementation was activated through its jar manifest"
(is (embeddings.provider/registered? "in-process"))
(is (= "true" (System/getProperty "ai.djl.offline")))
@@ -105,15 +163,15 @@
(is (nil? (find-ns 'metabase-enterprise.embedder.model))
"catalog access from an old worker thread remains lazy")))
(testing "the built artifact contains a ready, immutable model space"
- (is (true? (:ready? (embeddings.provider/readiness requested-model))))
+ (is (true? (:ready? (embeddings.provider/readiness retrieval-model))))
(is (= {:provider "in-process" :ready? false :reason :vector-dimensions-mismatch}
(embeddings.provider/readiness
- (assoc requested-model :vector-dimensions 768))))
+ (assoc retrieval-model :vector-dimensions 768))))
(is (re-matches #"emb:v1:sha256:[0-9a-f]{64}"
- (:embedding-space-id (embeddings.provider/resolve-model requested-model))))
+ (:embedding-space-id (embeddings.provider/resolve-model retrieval-model))))
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"has 384 dimensions"
(embeddings.provider/resolve-model
- (assoc requested-model :vector-dimensions 768))))
+ (assoc retrieval-model :vector-dimensions 768))))
(let [architecture-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)
os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)
@@ -123,7 +181,7 @@
os-var (constantly "linux")
libc-var (constantly :glibc)
glibc-version-var (constantly "2.34")}
- #(embeddings.provider/resolve-model requested-model)))
+ #(embeddings.provider/resolve-model retrieval-model)))
normal (mapv (comp :embedding-space-id resolve-for) ["arm64" "avx2"])
bounded (binding [*print-length* 1
*print-level* 1
@@ -135,66 +193,28 @@
"architecture-specific exports intentionally have distinct embedding spaces"))
(let [model-spec-var (ns-resolve 'metabase-enterprise.embedder.catalog 'model-spec)
model-spec-fn (var-get model-spec-var)
- spec (model-spec-fn (:model-name requested-model))
- normal (embeddings.provider/resolve-model requested-model)
+ spec (model-spec-fn (:model-name retrieval-model))
+ normal (embeddings.provider/resolve-model retrieval-model)
reordered (update spec :runtime #(into (array-map) (reverse %)))
equivalent (with-redefs-fn {model-spec-var (constantly reordered)}
- #(embeddings.provider/resolve-model requested-model))]
+ #(embeddings.provider/resolve-model retrieval-model))]
(is (= (:embedding-space-id normal) (:embedding-space-id equivalent))
"equivalent catalog map order does not change model identity")
- (check-missing-architecture! requested-model))
- (let [libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)
- os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
- detect-libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'detect-linux-libc)
- detect-libc (var-get detect-libc-var)
- glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)
- supported-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'supported-glibc-version?)
- supported-version? (var-get supported-version-var)]
- (is (= :unknown (detect-libc false nil)) "unreadable process maps fail closed")
- (is (= :musl (detect-libc true nil)))
- (is (= :musl (detect-libc false "/lib/ld-musl-x86_64.so.1")))
- (is (= :glibc (detect-libc false "/usr/lib64/ld-2.28.so")))
- (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2"))
- "the Ubuntu 24.04 loader layout is recognized")
- (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/libc.so.6")))
- (is (false? (supported-version? "2.33")))
- (is (true? (supported-version? "2.34")))
- (is (true? (supported-version? "2.39")))
- (is (false? (supported-version? nil)))
- (with-redefs-fn {libc-var (constantly :glibc)
- os-var (constantly "linux")
- glibc-version-var (constantly "2.34")}
- #(is (true? (:ready? (embeddings.provider/readiness requested-model)))))
- (with-redefs-fn {libc-var (constantly :unknown)
- os-var (constantly "linux")
- glibc-version-var (constantly "2.39")}
- #(is (true? (:ready? (embeddings.provider/readiness requested-model)))
- "the authoritative glibc API probe survives an inconclusive process map"))
- (with-redefs-fn {libc-var (constantly :glibc)
- os-var (constantly "linux")
- glibc-version-var (constantly "2.33")}
- #(do
- (is (= {:provider "in-process" :ready? false :reason :unsupported-libc-version}
- (embeddings.provider/readiness requested-model)))
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc-version"
- (embeddings.provider/resolve-model requested-model)))))
- (doseq [[unsupported-libc probed-glibc-version] [[:musl "2.39"]
- [:unknown nil]]]
- (with-redefs-fn {libc-var (constantly unsupported-libc)
- os-var (constantly "linux")
- glibc-version-var (constantly probed-glibc-version)}
- #(do
- (is (= {:provider "in-process" :ready? false :reason :unsupported-libc}
- (embeddings.provider/readiness requested-model)))
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc"
- (embeddings.provider/resolve-model requested-model)))))))
- (let [os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
- arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)]
- (with-redefs-fn {os-var (constantly "mac os x")
- arch-var (constantly "avx2")}
- #(is (= {:provider "in-process" :ready? false :reason :unsupported-platform}
- (embeddings.provider/readiness requested-model))
- "the tokenizer artifact has no Intel macOS native library"))))
+ (check-missing-architecture! retrieval-model))
+ (check-platform-support!))
+ (testing "the same artifact serves the complexity score's model in its own vector space"
+ (is (true? (:ready? (embeddings.provider/readiness complexity-model))))
+ (is (= {:provider "in-process" :ready? false :reason :vector-dimensions-mismatch}
+ (embeddings.provider/readiness (assoc complexity-model :vector-dimensions 1024))))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"has 384 dimensions"
+ (embeddings.provider/resolve-model
+ (assoc complexity-model :vector-dimensions 1024))))
+ (let [resolved (embeddings.provider/resolve-model complexity-model)]
+ (is (= 384 (:vector-dimensions resolved)))
+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}" (:embedding-space-id resolved)))
+ (is (not= (:embedding-space-id (embeddings.provider/resolve-model retrieval-model))
+ (:embedding-space-id resolved))
+ "the two bundled models must never share an embedding space")))
(testing "the artifact bounds local inference batches before loading the runtime"
(let [model-fn-var (ns-resolve 'metabase-enterprise.embedder.plugin 'model-fn)
batch-sizes (atom [])]
@@ -203,7 +223,7 @@
(fn [_model-name texts]
(swap! batch-sizes conj (count texts))
(mapv (fn [_] (float-array 384)) texts)))}
- #(embeddings.provider/embed-texts requested-model (repeat 65 "text")))
+ #(embeddings.provider/embed-texts retrieval-model (repeat 65 "text")))
(is (= [32 32 1] @batch-sizes))
(is (nil? (find-ns 'metabase-enterprise.embedder.model)))))
(testing "real inference runs through the plugin jar"
@@ -214,17 +234,23 @@
(let [[dog puppy invoice]
(await-thread!
(isolated-thread-call
- #(embeddings.provider/embed-texts requested-model ["dog" "puppy" "invoice"])))
- minilm-embeddings
- (await-thread!
- (isolated-thread-call
- #(embeddings.provider/embed-texts requested-minilm-model ["dog" "invoice"])))]
+ #(embeddings.provider/embed-texts retrieval-model ["dog" "puppy" "invoice"])))]
(is (some? (find-ns 'metabase-enterprise.embedder.model)))
(is (= [384 384 384] (mapv alength [dog puppy invoice])))
- (is (> (cosine dog puppy) (cosine dog invoice)))
- (is (= [384 384] (mapv alength minilm-embeddings)))
- (is (every? #(< (Math/abs (- 1.0 (magnitude %))) 1.0e-5) minilm-embeddings)
- "MiniLM produces normalized 384-dimensional embeddings"))
+ (is (> (cosine dog puppy) (cosine dog invoice))))
+ (testing "the complexity-score model infers from the same loaded artifact"
+ (let [[monthly-active-users daily-active-users invoice]
+ (await-thread!
+ (isolated-thread-call
+ #(embeddings.provider/embed-texts
+ complexity-model
+ ["monthly active users" "daily active users" "invoice"])))]
+ (is (= [384 384 384] (mapv alength [monthly-active-users daily-active-users invoice])))
+ (is (every? #(< (Math/abs (- 1.0 (magnitude %))) 1.0e-5)
+ [monthly-active-users daily-active-users invoice])
+ "MiniLM produces normalized embeddings")
+ (is (> (cosine monthly-active-users daily-active-users)
+ (cosine monthly-active-users invoice)))))
(finally
(ProxySelector/setDefault original-proxy)))
(is (empty? @network-attempts) "bundled inference performs no outbound requests"))))))
diff --git a/test/metabase/embeddings/embedder_plugin_name_test.clj b/test/metabase/embeddings/embedder_plugin_name_test.clj
index 6962c377463..cad310d8ff7 100644
--- a/test/metabase/embeddings/embedder_plugin_name_test.clj
+++ b/test/metabase/embeddings/embedder_plugin_name_test.clj
@@ -1,14 +1,14 @@
(ns metabase.embeddings.embedder-plugin-name-test
(:require
[clojure.test :refer :all]
- [metabase.core.core :as core]
+ [metabase.embeddings.startup :as embeddings.startup]
[metabase.util.yaml :as yaml]))
(set! *warn-on-reflection* true)
(deftest embedder-plugin-name-matches-manifest-test
- (testing "core/embedder-plugin-name is what plugins/registered? looks up by, so it must match the manifest"
+ (testing "embedder-plugin-name matches the manifest's info.name"
(is (= (get-in (yaml/parse-string
(slurp "modules/embedder/resources/metabase/embedder/metabase-plugin.yaml"))
[:info :name])
- core/embedder-plugin-name))))
+ embeddings.startup/embedder-plugin-name))))