The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use the in-process embedder for retrieval and complexity score (#78314)
Chris Truter285d779b78a9
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 4ca5a7173ef..a89e54a4394 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -992,8 +992,8 @@embeddings{:team "Metabot"- :api #{metabase.embeddings.provider}- :uses #{util}}+ :api #{metabase.embeddings.provider metabase.embeddings.startup}+ :uses #{plugins util}}entity-retrieval{:team "Metabot"@@ -3210,6 +3210,7 @@collectionsconfigdriver+ embeddingsmetabotmodelspremium-features@@ -3703,6 +3704,7 @@collectionsconfigconnection-pool+ embeddingseventshealth-inspectorlib-bediff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 5be40dd0617..d162cd8fa53 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -13,7 +13,7 @@:deprecated-var 84:discouraged-java-method 3:discouraged-namespace 80- :discouraged-var 107+ :discouraged-var 111:equals-true 1:main-without-gen-class 1:metabase/defsetting-namespace 2diff --git a/.github/file-paths.yaml b/.github/file-paths.yamlindex 18f9330ffd0..d3d1440482b 100644--- a/.github/file-paths.yaml+++ b/.github/file-paths.yaml@@ -232,7 +232,15 @@ embedder_plugin:- "bin/build/src/build_embedder_plugin.clj"- "deps.edn"- "modules/embedder/**"+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity_embedders.clj"+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/settings.clj"+ - "enterprise/backend/src/metabase_enterprise/data_complexity_score/synonym_source.clj"+ - "enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj"+ - "enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj"+ - "enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj"+ - "enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj"- "src/metabase/embeddings/provider.clj"+ - "src/metabase/embeddings/startup.clj"- "src/metabase/plugins/**"- "test/metabase/embeddings/embedder_plugin_artifact_test.clj"diff --git a/.github/workflows/build-scripts.yml b/.github/workflows/build-scripts.ymlindex 768770bce4c..2f87185af88 100644--- a/.github/workflows/build-scripts.yml+++ b/.github/workflows/build-scripts.yml@@ -33,6 +33,15 @@ jobs:timeout-minutes: 20permissions:contents: read+ services:+ postgres:+ image: pgvector/pgvector:pg17+ env:+ POSTGRES_USER: mb_test+ POSTGRES_DB: mb_test+ POSTGRES_HOST_AUTH_METHOD: trust+ ports:+ - 5432:5432steps:- uses: actions/checkout@v6- name: Prepare back-end environment@@ -45,6 +54,8 @@ jobs:jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase/embedder-model-catalog.ednjar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-arm64.zipjar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-avx2.zip+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/all-minilm-l6-v2-1110a243fdf4706b3f48f1d95db1a4f5529b4d41-arm64.zip+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/all-minilm-l6-v2-1110a243fdf4706b3f48f1d95db1a4f5529b4d41-avx2.zip- name: Run artifact-only plugin smoke testenv:MB_EMBEDDER_ARTIFACT_TEST: "true"@@ -52,6 +63,19 @@ jobs:run: >-./bin/test-agent --oss :only'[metabase.embeddings.embedder-plugin-artifact-test]'+ - name: Run in-process semantic-search round trip+ env:+ MB_DB_TYPE: postgres+ MB_DB_PORT: 5432+ MB_DB_HOST: localhost+ MB_DB_DBNAME: mb_test+ MB_DB_USER: mb_test+ MB_APPDB_PGVECTOR_MODE_TEST: "true"+ MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST: "true"+ MB_PLUGINS_DIR: modules/embedder/target+ run: >-+ ./bin/test-agent --drivers=postgres :only+ '[metabase-enterprise.semantic-search.appdb-pgvector-mode-test/appdb-pgvector-in-process-embedder-round-trip-test]'test-build-scripts:runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}diff --git a/bin/build/src/build/embedder_model.clj b/bin/build/src/build/embedder_model.cljindex 5ddcf03b11e..3aade9bd455 100644--- a/bin/build/src/build/embedder_model.clj+++ b/bin/build/src/build/embedder_model.clj@@ -1,5 +1,5 @@(ns build.embedder-model- "Fetch, verify, and package the pinned Arctic model bundles used by the embedder plugin."+ "Fetch, verify, and package the pinned model bundles used by the embedder plugin."(:require[clojure.edn :as edn][clojure.java.io :as io]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.cljindex f99fd8bb223..49e06eed433 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj@@ -40,6 +40,7 @@;; Loaded for side-effect: derives setting :on-change event topics from :metabase/event.;; metabase.core.core/entrypoint normally does this, but the standalone CLI bypasses it.[metabase.driver.init]+ [metabase.embeddings.startup :as embeddings.startup][metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -132,6 +133,9 @@Snowplow is off here, so we don't advance `data-complexity-scoring-last-fingerprint` — leave that to the cron."[write?](mdb/setup-db-without-migrations!)+ ;; `metabase.core.core/init!` does this on the server, but standalone modes skip it. Without it a synonym axis+ ;; configured for the `in-process` provider finds nothing registered and fails with `:provider-not-registered`.+ (embeddings.startup/ensure-in-process-provider!)(let [result (complexity/complexity-scores(assoc (synonym-source/complexity-scores-opts):metabot-scope (metabot-scope/internal-metabot-scope)diff --git a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj b/enterprise/backend/src/metabase_enterprise/embeddings/client.cljindex bb6cf1a547d..cdc96ac8a90 100644--- a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj+++ b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj@@ -31,22 +31,23 @@(set! *warn-on-reflection* true)-(defn- search-embedding-model+(defn- normalize-model-descriptor"Translate the neutral embedding descriptor's dimension key to semantic search's internal key."- [embedding-model]- (cond-> embedding-model- (and (nil? (:vector-dimensions embedding-model))- (some? (:model-dimensions embedding-model)))- (assoc :vector-dimensions (:model-dimensions embedding-model))))+ [{:keys [model-dimensions vector-dimensions] :as embedding-model}]+ (when (and model-dimensions vector-dimensions (not= model-dimensions vector-dimensions))+ (throw (ex-info "Embedding model descriptor has conflicting dimension values."+ {:model-dimensions model-dimensions+ :vector-dimensions vector-dimensions})))+ (cond-> (dissoc embedding-model :model-dimensions)+ model-dimensions (assoc :vector-dimensions model-dimensions)))(defn get-embeddings-batch"Return a sequential collection of embedding vectors, in the same order as the input texts.`embedding-model` — `{:provider :model-name :model-dimensions}` map; provider must be one of those registered on- [[metabase-enterprise.semantic-search.embedding/get-embeddings-batch]] (`ai-service`, `openai`,- `ollama`).+ [[metabase.embeddings.provider]] (`ai-service`, `openai`, `ollama`).`texts` — sequential collection of input strings.`opts` — optional keyword opts (e.g. `:type :doc`). Accepts alternating kwargs or a single trailing map;- forwarded as kwargs into the multimethod, which destructures with `& {:as opts}`."+ forwarded as kwargs to the provider."[embedding-model texts & {:as opts}]- (apply semantic-search/get-embeddings-batch (search-embedding-model embedding-model) texts (mapcat identity opts)))+ (apply semantic-search/get-embeddings-batch (normalize-model-descriptor embedding-model) texts (mapcat identity opts)))diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.cljindex c4e14f6a3ab..7c52699ebc4 100644--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj@@ -127,6 +127,22 @@(pgvector-configured?)(embedder-configured?)))+(defn- resolved-configured-model+ "Resolve the global setting to the immutable embedding space used by persisted library vectors."+ []+ (embedding/resolve-model (embedding/get-configured-model)))++(defn- resolved-configured-model-for-search+ []+ (try+ (resolved-configured-model)+ (catch clojure.lang.ExceptionInfo e+ ;; Readiness can change after the tool is offered (for example settings can select an unavailable model).+ ;; Domain/configuration failures should make this optional retrieval path disappear, while JVM errors and+ ;; other unexpected exception types still escape.+ (log/warn e "configured embedding model could not be resolved; returning no library results")+ nil)))+(defn- missing-table-error?"Whether we threw due to a Postgres undefined_table error (42P01), i.e. the indexes don't exist yet.Expected before the first build or after a manual drop awaiting heal.@@ -154,7 +170,7 @@[probe-populated?](try(let [ds (semantic.db.datasource/ensure-initialized-data-source!)- metadata-status (index-table/index-status ds (embedding/get-configured-model))+ metadata-status (index-table/index-status ds (resolved-configured-model))status (if (and (= :compatible metadata-status)(not (index-table/vectors-table-exists? ds))):missing@@ -277,7 +293,7 @@(let [ds (semantic.db.datasource/ensure-initialized-data-source!)waited-ms (elapsed-ms scheduled)started (u/start-timer)- diff (reconcile/reconcile! ds embedding/get-configured-model)+ diff (reconcile/reconcile! ds resolved-configured-model)ran-ms (elapsed-ms started)](record-run! "full" diff ran-ms){:index (select-keys diff [:inserted :deleted :unchanged])@@ -296,7 +312,7 @@(doseq [[entity-type entity-local-id :as entity-key] dirty](try(let [started (u/start-timer)- diff (reconcile/reconcile-entity! ds embedding/get-configured-model entity-type entity-local-id)]+ diff (reconcile/reconcile-entity! ds resolved-configured-model entity-type entity-local-id)](record-run! "targeted" diff (elapsed-ms started)))(catch Throwable e(log/error "library entity index: targeted reconcile failed; re-queuing"@@ -373,6 +389,15 @@(format "WHEN doc_type = '%s' THEN %s" (name k) w)))](format "(%s) - (CASE %s ELSE 0.0 END)" distance-expr cases)))+(defn- index-compatible?+ [connectable model]+ (try+ (= :compatible (index-table/index-status connectable model))+ (catch SQLException e+ (if (= "42P01" (.getSQLState e))+ false+ (throw e)))))+(defenterprise search-unfiltered"Find the library-entity documents best matching `user-search-prompt`, up to `limit`, ranked by ablended score (cosine similarity plus a slight doc_type bump).@@ -388,42 +413,57 @@[](let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)limit (or limit default-limit)- model (embedding/get-configured-model)- embedding (embedding/get-embedding model user-search-prompt- {:type :query :record-tokens? true})- lit (index-table/format-embedding embedding)- distance (str "doc_embedding <=> " lit)- rows (try- (jdbc/execute!- pgvector- (-> (sql.helpers/select :entity_type :entity_local_id :doc_type :doc_text- [[:raw distance] :distance])- (sql.helpers/from (keyword (index-table/vectors-table)))- ;; Exact scan, no HNSW: the blended order-by can't use an ANN index, and the- ;; library set is tiny.- (sql.helpers/order-by [[:raw (ranking-sql distance)] :asc])- (sql.helpers/limit limit)- (sql/format {:quoted true}))- {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (catch SQLException e- ;; Only the index-not-ready states degrade to no results — the index, not the agent,- ;; is at fault and the next reconcile heals it:- ;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet)- ;; 22000 stored vectors incompatible with the query vector (a dimension/format change- ;; awaiting rebuild — e.g. a vector(OLD) column vs a vector(NEW) literal)- ;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool- ;; reports the search as unavailable rather than as an empty library.- (case (.getSQLState e)- "42P01" []- "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))- [])- (do (analytics/inc! :metabase-entity-retrieval/search-failed)- (throw e)))))]- (->> rows- (map (fn [row]- {:entity {:model (:entity_type row) :id (:entity_local_id row)}- :doc_type (:doc_type row)- :doc_text (:doc_text row)- :score (score row)}))- (sort-by (comp :total_score :score) >)- vec))))+ model (resolved-configured-model-for-search)]+ ;; Check compatibility under a short non-blocking read lock. Even metadata-table DDL can otherwise block this+ ;; preflight behind reconcile before we reach the protected query path. The later locked recheck remains+ ;; authoritative because a reconcile can begin while the embedding is generated.+ (if-not (and model+ (true? (reconcile/with-index-read-lock pgvector #(index-compatible? % model))))+ []+ (let [embedding (embedding/get-embedding model (embedding/prefix-search-query model user-search-prompt)+ {:type :query :record-tokens? true})+ lit (index-table/format-embedding embedding)+ distance (str "doc_embedding <=> " lit)]+ (or+ (reconcile/with-index-read-lock+ pgvector+ (fn [conn]+ ;; The compatibility check and query share the reconcile lock and connection. A setting change+ ;; after the tool was offered must never query same-width vectors from another immutable space.+ (if-not (index-compatible? conn model)+ []+ (let [rows (try+ (jdbc/execute!+ conn+ (-> (sql.helpers/select :entity_type :entity_local_id :doc_type :doc_text+ [[:raw distance] :distance])+ (sql.helpers/from (keyword (index-table/vectors-table)))+ ;; Exact scan, no HNSW: the blended order-by can't use an ANN index, and the+ ;; library set is tiny.+ (sql.helpers/order-by [[:raw (ranking-sql distance)] :asc])+ (sql.helpers/limit limit)+ (sql/format {:quoted true}))+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (catch SQLException e+ ;; Only the index-not-ready states degrade to no results — the index, not the agent,+ ;; is at fault and the next reconcile heals it:+ ;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet)+ ;; 22000 stored vectors incompatible with the query vector (a dimension/format change+ ;; awaiting rebuild — e.g. a vector(OLD) column vs a vector(NEW) literal)+ ;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool+ ;; reports the search as unavailable rather than as an empty library.+ (case (.getSQLState e)+ "42P01" []+ "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))+ [])+ (do (analytics/inc! :metabase-entity-retrieval/search-failed)+ (throw e)))))]+ (->> rows+ (map (fn [row]+ {:entity {:model (:entity_type row) :id (:entity_local_id row)}+ :doc_type (:doc_type row)+ :doc_text (:doc_text row)+ :score (score row)}))+ (sort-by (comp :total_score :score) >)+ vec)))))+ []))))))diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.cljindex 022c2c85bcb..2f3b4d52150 100644--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj@@ -52,7 +52,7 @@:meta (str index-schema ".library_entity_index_meta")})(def legacy-tables- "The unqualified names used before [[index-schema]] existed. [[adopt-legacy-tables!]] moves them."+ "The unqualified names used before [[index-schema]] and immutable embedding-space identities existed."{:vectors "library_entity_index":meta "library_entity_index_meta"})@@ -88,17 +88,17 @@(semantic.util/quote-table (meta-table)))(def schema-version- "Canonical version of the index's *document format* — both the vectors table schema and the+ "Canonical version of the persisted index contract — the metadata and vectors table schemas plus thedoc-derivation contract (doc_id scheme, doc_type set, doc_text source, dedup/key rules).It's part of the meta row's [[model-identity]], so bumping it makes [[ensure-tables!]] drop and rebuildthe vectors table; the post-upgrade startup reconcile then repopulates from the appdb under the new- format. Bump on ANY format-affecting change in [[metabase-enterprise.entity-retrieval.reconcile]] or- the table schema: a vectors-table column/type change, a new or renamed doc_type, a changed doc_text+ format. Bump on ANY compatibility-affecting change in [[metabase-enterprise.entity-retrieval.reconcile]]+ or either table schema: a column/type change, a new or renamed doc_type, a changed doc_textsource, or a changed doc_id / dedup / key scheme — anything that makes old rows incomparable to newlyderived desired docs. A bump forces a full re-embed of the library on every instance at upgrade, so doit only when the format truly moved, never as a refresh convenience."- ;; v1 — initial schema.- 1)+ ;; v1 — initial schema; v2 — immutable embedding-space identity.+ 2);; Advisory lock serializing concurrent ensure-tables! calls (e.g. several cluster nodes starting at;; once). Arbitrary app-wide-unique constant; semantic-search's migration lock uses 19991.@@ -124,6 +124,7 @@[:provider :text :not-null][:model_name :text :not-null][:vector_dimensions :int :not-null]+ [:embedding_space_id :text :not-null][:schema_version :int :not-null][:updated_at :timestamp-with-time-zone :not-null];; Captured immediately before a successful full reconcile reads the appdb. Distinct from updated_at@@ -149,27 +150,45 @@{:provider (:provider embedding-model):model_name (:model-name embedding-model):vector_dimensions (:vector-dimensions embedding-model)+ :embedding_space_id (:embedding-space-id embedding-model):schema_version schema-version})-(defn- read-meta [tx]- (jdbc/execute-one! tx- [(format "SELECT provider, model_name, vector_dimensions, schema_version FROM %s WHERE id = 1"- (meta-table-sql))]- {:builder-fn jdbc.rs/as-unqualified-lower-maps}))+(defn- meta-column-exists? [tx column]+ (some? (jdbc/execute-one! tx [(str "SELECT 1 FROM pg_attribute"+ " WHERE attrelid = to_regclass(?) AND attname = ? AND NOT attisdropped")+ (meta-table) column])))++(defn- read-meta+ "The meta row shaped like [[model-identity]], or nil when nothing is built.++ A v1 table has no `embedding_space_id` column. Report it as nil instead of failing the SELECT, so the+ identity comparison in [[index-status]] / [[ensure-tables!]] sees the mismatch and schedules the rebuild+ that fills it in."+ [tx]+ (let [space? (meta-column-exists? tx "embedding_space_id")]+ (some->> (jdbc/execute-one! tx+ [(format (str "SELECT provider, model_name, vector_dimensions, schema_version%s"+ " FROM %s WHERE id = 1")+ (if space? ", embedding_space_id" "")+ (meta-table-sql))]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (merge {:embedding_space_id nil}))))(defn- write-meta! [tx embedding-model]- (let [{:keys [provider model_name vector_dimensions schema_version]} (model-identity embedding-model)]+ (let [{:keys [provider model_name vector_dimensions embedding_space_id schema_version]}+ (model-identity embedding-model)](jdbc/execute! tx(-> (sql.helpers/insert-into (keyword (meta-table)))(sql.helpers/values [{:id 1:provider provider:model_name model_name:vector_dimensions vector_dimensions+ :embedding_space_id embedding_space_id:schema_version schema_version:updated_at (Instant/now)}])(sql.helpers/on-conflict :id)(sql.helpers/do-update-set :provider :model_name :vector_dimensions- :schema_version :updated_at)+ :embedding_space_id :schema_version :updated_at)sql-format-quoted))))(defn- create-tables! [tx dims]@@ -196,36 +215,45 @@e))))));; TODO (Chris 2026-08-17) -- delete this once no supported upgrade can start from the bare names.-(defn- adopt-legacy-tables!- "Move an index built before [[index-schema]] existed into it, preserving its rows.- Without this, the qualified tables would be created empty beside the old ones and the whole library- would be re-embedded. A no-op once moved: the unqualified name no longer resolves.-- A dedicated store only. The bare names date from when this index was dedicated-only, so nothing in an- app db can be an old index of ours -- but [[table-exists?]] resolves an unqualified name on the search- path, where an application table that happens to share the name would answer, and get moved."+(defn- legacy-table-in-search-path+ "The first table named `table-name` on the effective search path, excluding the current index schema."+ [tx table-name]+ (when-let [schema-name (:schema_name+ (jdbc/execute-one!+ tx+ [(str "SELECT n.nspname AS schema_name"+ " FROM unnest(current_schemas(true)) WITH ORDINALITY AS p(schema_name, search_order)"+ " JOIN pg_namespace n ON n.nspname = p.schema_name"+ " JOIN pg_class c ON c.relnamespace = n.oid"+ " WHERE c.relname = ? AND c.relkind IN ('r', 'p') AND n.nspname <> ?"+ " ORDER BY p.search_order LIMIT 1")+ table-name index-schema]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))]+ {:schema-name schema-name :table-name table-name}))++(defn- drop-legacy-tables!+ "Drop an index built before [[index-schema]] and immutable embedding-space identities existed.++ Its vectors have no trustworthy space identity, so they must be rebuilt rather than moved into the+ current schema. A dedicated store only: an app db may contain unrelated application tables with these+ names. Resolve the effective search path, excluding the current index schema, then drop the resolved+ tables by qualified name."[tx](when (and (= default-tables (tables))(semantic.db.datasource/dedicated-url-configured?))(doseq [k [:vectors :meta]]- (let [legacy (k legacy-tables)]- (when (and (table-exists? tx legacy)- (not (table-exists? tx (k default-tables))))- (log/infof "Moving %s into the %s schema" legacy index-schema)- (jdbc/execute! tx [(format "ALTER TABLE %s SET SCHEMA %s"- (semantic.util/quote-ident legacy)- (semantic.util/quote-ident index-schema))]))))))+ (when-let [{:keys [schema-name table-name]} (legacy-table-in-search-path tx (k legacy-tables))]+ (let [legacy-sql (format "%s.%s"+ (semantic.util/quote-ident schema-name)+ (semantic.util/quote-ident table-name))]+ (log/infof "Dropping incompatible legacy library entity index table %s.%s" schema-name table-name)+ (jdbc/execute! tx [(format "DROP TABLE %s" legacy-sql)]))))))(defn vectors-table-exists?"Whether the configured entity-retrieval vectors table exists."[tx](table-exists? tx (vectors-table)))-(defn- meta-column-exists? [tx column]- (some? (jdbc/execute-one! tx [(str "SELECT 1 FROM pg_attribute"- " WHERE attrelid = to_regclass(?) AND attname = ? AND NOT attisdropped")- (meta-table) column])))-(defn- can-alter-meta-table? [tx](:owns_table(jdbc/execute-one! tx@@ -271,6 +299,38 @@(jdbc/execute! tx [(format "UPDATE %s SET reconciled_at = NULL WHERE id = 1"(meta-table-sql))])))+(defn- embedding-space-column-state+ "Whether the meta table's `embedding_space_id` column is `:missing`, `:nullable`, or `:not-null`."+ [tx]+ (if-let [row (jdbc/execute-one! tx+ [(str "SELECT attnotnull FROM pg_attribute"+ " WHERE attrelid = to_regclass(?) AND attname = 'embedding_space_id'"+ " AND NOT attisdropped")+ (meta-table)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (if (:attnotnull row) :not-null :nullable)+ :missing))++(defn- ensure-embedding-space-column!+ "Add `embedding_space_id` to a meta table built before it existed. Returns the column's prior state.++ Unlike `reconciled_at`, this column is load-bearing: [[read-meta]] reads it and [[model-identity]]+ compares it, so a role that cannot add it can never bring the index to a compatible state. Raise a typed+ error naming the ownership problem rather than letting a bare SQLSTATE 42501 abort the caller's+ transaction — [[metabase-enterprise.entity-retrieval.core/retrieval-status]] reports it as `:unreachable`+ and the retrieval tool stops being offered until an admin fixes the grant."+ [tx]+ (let [state (embedding-space-column-state tx)]+ (when (= :missing state)+ (when-not (can-alter-meta-table? tx)+ (throw (ex-info (format (str "Cannot add embedding_space_id to %s: the current role does not own it."+ " Grant ownership of the library entity index tables to the Metabase"+ " user, or drop them so they can be recreated.")+ (meta-table))+ {:type ::embedding-space-upgrade-failed, :table (meta-table)})))+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN embedding_space_id TEXT" (meta-table-sql))]))+ state))+(defn index-status"Compatibility of the built index against `embedding-model` and [[schema-version]]:- `:missing` no meta row — nothing built@@ -299,41 +359,49 @@(jdbc/execute! tx [(format "SELECT pg_advisory_xact_lock(%d)" ensure-lock-id)])(jdbc/execute! tx (sql/format (sql.helpers/create-extension :vector :if-not-exists)))(ensure-schema! tx)- ;; Before any CREATE below: those are IF NOT EXISTS, so creating first would leave an empty qualified- ;; table for the move to refuse, and the whole library would be re-embedded.- (adopt-legacy-tables! tx)+ ;; Before any CREATE below: remove the pre-embedding-space tables whose vectors cannot be trusted.+ (drop-legacy-tables! tx)(jdbc/execute! tx (create-meta-table-sql));; CREATE TABLE IF NOT EXISTS does not add columns to an existing table. Upgrade when this role owns it;;; grant-only roles keep reconciling without the optional freshness timestamp.(ensure-reconciled-at-column! tx)- (let [stored (read-meta tx)- current (model-identity embedding-model)- dims (:vector_dimensions current)- result (cond- (nil? stored)- (do (create-tables! tx dims)- (write-meta! tx embedding-model)- :created)-- (not= stored current)- (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS %s"- (vectors-table-sql))])- (create-tables! tx dims)- (write-meta! tx embedding-model)- :rebuilt)-- :else- ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table- ;; heals itself, and report :created when it had actually gone missing — the recreated- ;; table is empty, so a targeted reconcile must repopulate the whole library rather than- ;; fill it with one entity.- (let [existed? (vectors-table-exists? tx)]- (create-tables! tx dims)- (if existed? :ok :created)))]+ (let [space-column (ensure-embedding-space-column! tx)+ stored (read-meta tx)+ current (model-identity embedding-model)+ dims (:vector_dimensions current)+ result (cond+ (nil? stored)+ (do (create-tables! tx dims)+ (write-meta! tx embedding-model)+ :created)++ (not= stored current)+ (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS %s"+ (vectors-table-sql))])+ (create-tables! tx dims)+ (write-meta! tx embedding-model)+ :rebuilt)++ :else+ ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table+ ;; heals itself, and report :created when it had actually gone missing — the recreated+ ;; table is empty, so a targeted reconcile must repopulate the whole library rather+ ;; than fill it with one entity.+ (let [existed? (vectors-table-exists? tx)]+ (create-tables! tx dims)+ (if existed? :ok :created)))];; Any empty (re)build invalidates reconciled_at: the table is empty until a full reconcile repopulates;; it, and neither write-meta! (:rebuilt) nor the heal branch touches reconciled_at, so a prior build's;; timestamp would linger and make NLQ staleness read fresh over an empty/incomplete index. Clear it;;; touch-reconciled-at! (converged reconciles only) sets it again once the index is actually verified.(when (not= result :ok)(clear-reconciled-at! tx))+ ;; Every row carries a space id by now — the mismatch above forced the rebuild that wrote it — so the+ ;; constraint can go on. SET NOT NULL takes an ACCESS EXCLUSIVE lock and needs ownership, so skip it+ ;; when it is already there or the role cannot alter the table; the identity compare, not the+ ;; constraint, is what keeps a stale index from being served.+ (when (and (not= :not-null space-column)+ (can-alter-meta-table? tx))+ (jdbc/execute! tx [(format "ALTER TABLE %s ALTER COLUMN embedding_space_id SET NOT NULL"+ (meta-table-sql))]))result)))diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.cljindex 54af8e03d50..3621b37da52 100644--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj@@ -50,6 +50,47 @@;; semantic-search's migration lock (19991).(def ^:private reconcile-lock-id 20012)+(defn- with-index-lock+ [pgvector lock-function unlock-function f]+ (with-open [^Connection conn (jdbc/get-connection pgvector)]+ (let [autocommit (.getAutoCommit conn)]+ (.setAutoCommit conn true)+ (try+ (jdbc/execute! conn [(format "SELECT %s(%d)" lock-function reconcile-lock-id)])+ (try+ (f conn)+ (finally+ (jdbc/execute! conn [(format "SELECT %s(%d)" unlock-function reconcile-lock-id)])))+ (finally+ (.setAutoCommit conn autocommit))))))++(defn with-index-read-lock+ "Run `(f connection)` under a shared cluster-wide library-index lock, returning nil immediately when a+ reconcile holds the matching exclusive lock.++ Concurrent searches share this lock. The non-blocking acquisition prevents waiting searches from exhausting+ the pgvector connection pool during a long reconcile. A successful search's compatibility check and vector+ query cannot straddle a rebuild into another embedding space."+ [pgvector f]+ (with-open [^Connection conn (jdbc/get-connection pgvector)]+ (let [autocommit (.getAutoCommit conn)]+ (.setAutoCommit conn true)+ (try+ (when (:acquired (jdbc/execute-one!+ conn+ [(format "SELECT pg_try_advisory_lock_shared(%d) AS acquired" reconcile-lock-id)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))+ (try+ (f conn)+ (finally+ (jdbc/execute! conn [(format "SELECT pg_advisory_unlock_shared(%d)" reconcile-lock-id)]))))+ (finally+ (.setAutoCommit conn autocommit))))))++(defn- with-index-write-lock+ [pgvector f]+ (with-index-lock pgvector "pg_advisory_lock" "pg_advisory_unlock" f))+(defn doc-id"Content-addressed primary key for an index document.`instructions` is intentionally not an input: editing it must not re-embed an entity's name/synonyms."@@ -416,28 +457,19 @@`ensure-tables!` (which may drop+rebuild on a model/format change) runs under the lock too, so a rebuildcan't pull the table out from under a concurrent node's in-flight run."[pgvector resolve-model f]- (with-open [^Connection conn (jdbc/get-connection pgvector)]- ;; Per-batch commits, not one big transaction: the run tolerates partial failure across runs, so each- ;; insert/delete must commit on its own. Some pools hand out autocommit-off connections (which would- ;; silently roll the whole run back on close), so force it on and restore the prior setting before the- ;; connection goes back to the pool. (ensure-tables! flips this to a transaction for its DDL.)- (let [autocommit (.getAutoCommit conn)]- (.setAutoCommit conn true)- (try- (jdbc/execute! conn [(format "SELECT pg_advisory_lock(%d)" reconcile-lock-id)])- (try- (let [embedding-model (resolve-model)- status (index-table/ensure-tables! conn embedding-model)- ;; :created (first build / healed manual drop) and :rebuilt (model/format change) both leave- ;; an empty table, so a targeted caller must do a full repopulate rather than index one entity.- emptied? (contains? #{:created :rebuilt} status)]- (when emptied?- (log/info "library entity index: vectors table is empty (" status "); repopulating"))- (assoc (f conn embedding-model emptied?) :rebuilt? (= :rebuilt status)))- (finally- (jdbc/execute! conn [(format "SELECT pg_advisory_unlock(%d)" reconcile-lock-id)])))- (finally- (.setAutoCommit conn autocommit))))))+ (with-index-write-lock+ pgvector+ (fn [conn]+ ;; Per-batch commits, not one big transaction: the run tolerates partial failure across runs. The+ ;; shared lock helper forces autocommit on; ensure-tables! temporarily wraps its DDL in a transaction.+ (let [embedding-model (resolve-model)+ status (index-table/ensure-tables! conn embedding-model)+ ;; :created (first build / healed manual drop) and :rebuilt (model/format change) both leave+ ;; an empty table, so a targeted caller must do a full repopulate rather than index one entity.+ emptied? (contains? #{:created :rebuilt} status)]+ (when emptied?+ (log/info "library entity index: vectors table is empty (" status "); repopulating"))+ (assoc (f conn embedding-model emptied?) :rebuilt? (= :rebuilt status))))))(defn reconcile!"Full reconcile of the pgvector `library_entity_index` with the appdb, blocking until it completes;diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.cljindex ce43fba805a..5b355584d85 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj@@ -83,7 +83,7 @@(do(log/infof "Starting migration from version %d to %d."db-version semantic.db.migration.impl/schema-version)- (semantic.db.migration.impl/migrate-schema! tx opts)+ (semantic.db.migration.impl/migrate-schema! tx (assoc opts :from-version db-version))(write-successful-migration! index-metadata tx)):elsediff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex 810e3f12b66..8f1b0957ae1 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -1,11 +1,13 @@(ns metabase-enterprise.semantic-search.db.migration.impl(:require+ [clojure.string :as str][honey.sql :as sql][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.util :as semantic.util][metabase.collections.curation :as collections.curation][metabase.config.core :as config]+ [metabase.embeddings.provider :as embeddings.provider][metabase.util :as u][metabase.util.log :as log][next.jdbc :as jdbc]@@ -15,7 +17,7 @@(def schema-version"Version to compare the [[metabase-enterprise.semantic-search.db.migration/db-version]] with. If this is higher,schema migration will be performed."- 2)+ 3)(def ^:private app-db-sentinel-tables"Tables specific to a Metabase application database, chosen to avoid generic names (e.g. Liquibase's@@ -91,14 +93,56 @@{:drop-table [[[:raw (str (semantic.util/quote-ident schemaname) "."(semantic.util/quote-ident tablename))]]]})))))+(defn- quoted-table-name+ [table-name]+ (->> (semantic.util/qualified-table-parts table-name)+ (remove nil?)+ (map semantic.util/quote-ident)+ (str/join ".")))++(defn- add-embedding-space-metadata!+ "Migration 3: preserve existing semantic indexes and add immutable embedding-space metadata."+ [tx index-metadata]+ (let [table-name (:metadata-table-name index-metadata)+ control-table-name (:control-table-name index-metadata)+ table (keyword table-name)]+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN IF NOT EXISTS embedding_space_id TEXT"+ (quoted-table-name table-name))])+ ;; Legacy rows have no trustworthy revision provenance. Keep this nullable and let their backfilled+ ;; embedding-space ID represent the old, explicitly unspecified revision.+ (jdbc/execute! tx [(format "ALTER TABLE %s ADD COLUMN IF NOT EXISTS model_revision TEXT"+ (quoted-table-name table-name))])+ (doseq [{:keys [id provider model_name vector_dimensions]}+ (jdbc/execute! tx+ (sql/format {:select [:id :provider :model_name :vector_dimensions]+ :from [table]})+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (let [space-id (:embedding-space-id+ (embeddings.provider/legacy-resolved-model+ {:provider provider :model-name model_name :vector-dimensions vector_dimensions}))]+ (jdbc/execute! tx+ (sql/format {:update table+ :set {:embedding_space_id space-id}+ :where [:= :id id]}))))+ (jdbc/execute! tx [(format "ALTER TABLE %s ALTER COLUMN embedding_space_id SET NOT NULL"+ (quoted-table-name table-name))])+ (jdbc/execute! tx+ (sql/format {:update (keyword control-table-name)+ :set {:version (:version index-metadata)}+ :where [:= :id 0]}+ :quoted true))))+(defn migrate-schema!"Migrate schema (control, metadata, gate, ...). Migration author is responsible for removing leftovers if necessaryand in general leaving schema in desired state."- [tx {:keys [index-metadata] :as _opts}]- ;; ideally index_table indexed are manipulated in dynamic schema part but for now it does not matter- (drop-all-but-migration-table index-metadata tx)- (semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)- (semantic.index-metadata/ensure-control-row-exists! tx index-metadata))+ [tx {:keys [index-metadata from-version] :as _opts}]+ (if (= 2 from-version)+ (add-embedding-space-metadata! tx index-metadata)+ (do+ ;; Older versions retain the existing reset migration. Only v2 has enough metadata to backfill safely.+ (drop-all-but-migration-table index-metadata tx)+ (semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)+ (semantic.index-metadata/ensure-control-row-exists! tx index-metadata))))(defn ensure-schema-compatibility!"Apply non-destructive additions that existing databases at the current schema version also need."diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.cljindex 509471f96f1..aecd4f33aef 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj@@ -1,5 +1,7 @@(ns metabase-enterprise.semantic-search.embedding(:require+ [buddy.core.codecs :as buddy-codecs]+ [buddy.core.hash :as buddy-hash][clj-http.client :as http][clojure.string :as str][diehard.circuit-breaker :as dh.cb]@@ -8,6 +10,7 @@[metabase-enterprise.semantic-search.settings :as semantic-settings][metabase.analytics-interface.core :as analytics][metabase.analytics.core :as analytics.core]+ [metabase.embeddings.provider :as embeddings.provider][metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features][metabase.tracing.core :as tracing]@@ -46,14 +49,23 @@(defn clean-provider-name"Clean up a provider names for use in index names."[provider-name]- (str/replace provider-name #"[-:.]" "_"))+ (-> provider-name+ (str/replace #"[^A-Za-z0-9_]" "_")+ (str/replace #"_{2,}" "_")+ (str/replace #"^_+|_+$" "")))(defn abbrev-provider-name"Abbreviate long provider names for use in index names."[provider-name](case provider-name"ai-service" "ais"- (clean-provider-name provider-name)))+ "ollama" "ollama"+ "openai" "openai"+ "in-process" "inproc"+ ;; Every plugin provider uses the same reserved form, including already-safe names. Otherwise an unsafe name's+ ;; encoded output could itself be registered as a safe provider name and collide with the same physical index.+ (str "plugin_" (clean-provider-name provider-name) "_"+ (subs (buddy-codecs/bytes->hex (buddy-hash/sha1 provider-name)) 0 16))));;; Token Counting for OpenAI Models@@ -133,23 +145,57 @@(conj batches current-batch)batches)))-;;;; Provider SPI--(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))+;;;; Provider API-(defmulti get-embedding- "Returns a single embedding vector for the given text.- `opts` (kwargs, honoured by the production providers; ollama ignores them):+(defn- record-in-process-token-usage!+ [{:keys [provider model-name]} texts {:keys [record-tokens? type]}]+ (when (= "in-process" provider)+ ;; The bundled tokenizer is intentionally private to the plugin artifact. Use the existing cl100k+ ;; counter as an operational estimate so local inference participates in the same metrics/tracking.+ (let [tokens (count-tokens-batch texts)]+ (analytics/inc! :metabase-search/semantic-embedding-tokens+ {:provider provider :model model-name}+ tokens)+ (when record-tokens?+ (semantic.models.token-tracking/record-tokens model-name type tokens)))))++(defn resolve-model+ "Resolve a requested embedding model to its immutable vector-space descriptor."+ [embedding-model]+ (embeddings.provider/resolve-model embedding-model))++(defn- in-process-token-model+ [{:keys [provider model-name] :as embedding-model}]+ (when (= "in-process" provider)+ ;; Configured and persisted descriptors already carry the canonical name. Only the optional plugin default+ ;; needs early resolution so token tracking has a stable label; the provider still owns embed-time resolution.+ (if (nil? model-name)+ (resolve-model embedding-model)+ embedding-model)))++(defn get-embedding+ "Return one embedding vector for `text`.+ `opts` (honoured by the production providers; ollama ignores them):- `:record-tokens?` — write a token-tracking row for the call- `:type` — what the embedding is for (`:query`/`:index`), recorded with the tokens- `:snowplow?` — ai-service only, default true; synthetic callers (e.g. the health probe) passfalse so the call emits no token_usage event"- {:arglists '([embedding-model text & opts])} dispatch-provider)--(defmulti get-embeddings-batch- "Returns a sequential collection of embedding vectors, in the same order as the input texts.+ [embedding-model text & {:as opts}]+ (let [token-model (in-process-token-model embedding-model)]+ (u/prog1 (embeddings.provider/embed-text (or token-model embedding-model) text opts)+ (when token-model+ (record-in-process-token-usage! token-model [text] opts)))))++(defn get-embeddings-batch+ "Return one embedding vector per input text, in the same order.Takes the same `opts` as [[get-embedding]], minus `:snowplow?` (batch callers are all organic)."- {:arglists '([embedding-model texts & opts])} dispatch-provider)+ [embedding-model texts & {:as opts}]+ (let [token-model (in-process-token-model embedding-model)]+ (u/prog1 (embeddings.provider/embed-texts (or token-model embedding-model) texts opts)+ (when token-model+ (record-in-process-token-usage! token-model texts opts)))))++(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))(defmulti embedder-circuit-endpoint"Return the remote endpoint that identifies `embedding-model`'s circuit, or nil when its provider does not@@ -158,9 +204,10 @@(defmethod embedder-circuit-endpoint :default [_] nil)-(defmulti pull-model- "If a model needs to be downloaded (which is the case for ollama), downloads it."- {:arglists '([embedding-model])} dispatch-provider)+(defn pull-model+ "Prepare a provider model eagerly when the provider supports it."+ [embedding-model]+ (embeddings.provider/prepare! embedding-model));;;; Embedding-service circuit breaker;;;@@ -170,6 +217,10 @@;;; failing. State transitions surface the affected health checks immediately (see the listeners) rather;;; than waiting for the daily job. Thresholds are fixed (the breaker is built once); the setting is a;;; runtime kill switch checked per call.+;;;+;;; The breaker wraps the HTTP providers defined below rather than sitting behind the provider SPI, since+;;; it keys circuits by remote endpoint and an in-process provider has none. If a second provider that+;;; needs it turns up, move it into `metabase.embeddings.provider` instead of adding a second copy.(def ^:private embedder-circuit-breaker-failure-threshold"Consecutive embedding-service failures that trip the breaker open." 5)@@ -385,11 +436,7 @@(throw e))));; Ollama is not used in production. Token tracking is not implemented.-(defmethod get-embedding "ollama" [{:keys [model-name vector-dimensions]} text & {:as _opts}]- (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)- :endpoint ollama-embeddings-endpoint))--(defmethod get-embeddings-batch "ollama" [{:keys [model-name vector-dimensions]} texts & {:as _opts}]+(defn- ollama-get-embeddings-batch [model-name vector-dimensions texts];; Ollama doesn't have a native batch API, so we fall back to individual calls. Each call goes through the;; breaker on its own, so an outage mid-batch fast-fails the remaining texts instead of timing out on each.(log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")@@ -398,8 +445,6 @@:endpoint ollama-embeddings-endpoint))texts))-(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))-;;;; OpenAI-compatible embedding service impl (shared by "ai-service" and "openai" providers)(defn- supports-dimensions?@@ -528,22 +573,8 @@(defmethod embedder-circuit-endpoint "ai-service" [_](first (embedding-service-resolve-config!)))-(defmethod get-embedding "ai-service"- [{:keys [model-name vector-dimensions]} text & {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]- (let [[endpoint api-key] (embedding-service-resolve-config!)]- (first (openai-compatible-get-embeddings-batch- {:provider "ai-service"- :endpoint endpoint- :api-key api-key- :model-name model-name- :vector-dimensions vector-dimensions- :texts [text]- :snowplow? snowplow?- :record-tokens? record-tokens?- :type type}))))--(defmethod get-embeddings-batch "ai-service"- [{:keys [model-name vector-dimensions]} texts & {:keys [record-tokens? type]}]+(defn- ai-service-get-embeddings-batch+ [{:keys [model-name vector-dimensions]} texts {:keys [record-tokens? type snowplow?] :or {snowplow? true}}](let [[endpoint api-key] (embedding-service-resolve-config!)](openai-compatible-get-embeddings-batch{:provider "ai-service"@@ -552,13 +583,10 @@:model-name model-name:vector-dimensions vector-dimensions:texts texts- :snowplow? true+ :snowplow? snowplow?:record-tokens? record-tokens?:type type})))-(defmethod pull-model "ai-service" [_]- (log/debug "ai-service provider does not require pulling a model"))-;;;; OpenAI provider(defn- openai-resolve-config!@@ -572,23 +600,8 @@(defmethod embedder-circuit-endpoint "openai" [_](first (openai-resolve-config!)))-(defmethod get-embedding "openai"- [embedding-model text & {:keys [record-tokens? type]}]- (let [[endpoint api-key] (openai-resolve-config!)]- (first (openai-compatible-get-embeddings-batch- {:provider "openai"- :endpoint endpoint- :api-key api-key- :model-name (:model-name embedding-model)- :vector-dimensions (:vector-dimensions embedding-model)- :texts [text]- :record-tokens? record-tokens?- :extra-body (when (supports-dimensions? embedding-model)- {:dimensions (:vector-dimensions embedding-model)})- :type type}))))--(defmethod get-embeddings-batch "openai"- [embedding-model texts & {:keys [record-tokens? type]}]+(defn- openai-get-embeddings-batch+ [embedding-model texts {:keys [record-tokens? type]}](let [[endpoint api-key] (openai-resolve-config!)](openai-compatible-get-embeddings-batch{:provider "openai"@@ -602,18 +615,50 @@{:dimensions (:vector-dimensions embedding-model)}):type type})))-(defmethod pull-model "openai" [_]- (log/debug "OpenAI provider does not require pulling a model"))+(defn- register-built-in-providers!+ []+ (let [spi-version embeddings.provider/embedding-spi-version+ legacy embeddings.provider/legacy-resolved-model]+ (embeddings.provider/register-provider!+ "ollama"+ {:embedding-spi-version spi-version+ :readiness (constantly {:ready? true})+ :resolve-model legacy+ :embed-texts (fn [{:keys [model-name vector-dimensions]} texts _opts]+ (ollama-get-embeddings-batch model-name vector-dimensions texts))+ :prepare! (fn [{:keys [model-name]}]+ (ollama-pull-model model-name))})+ (embeddings.provider/register-provider!+ "ai-service"+ {:embedding-spi-version spi-version+ :readiness (fn [_]+ {:ready? (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))+ (not-empty (llm.settings/ai-service-base-url))))})+ :resolve-model legacy+ :embed-texts ai-service-get-embeddings-batch})+ (embeddings.provider/register-provider!+ "openai"+ {:embedding-spi-version spi-version+ :readiness (fn [_]+ {:ready? (boolean (not-empty (semantic-settings/openai-api-key)))})+ :resolve-model legacy+ :embed-texts openai-get-embeddings-batch})))++(register-built-in-providers!);;;; Query prefixes for asymmetric retrieval models(def ^:private model-family-query-prefixes"Query prefixes for embedding-model families trained for asymmetric retrieval.- These models expect search queries — but not the indexed documents — to carry a fixed prefix."- ;; Patterns must be mutually exclusive: lookup scans entries in unspecified order.+ These models expect search queries — but not the indexed documents — to carry a fixed prefix.+ First match wins, so keep more specific patterns first."+ ;; Arctic Embed moved to the short "query: " prefix at v2.0. Everything earlier expects the original+ ;; longer instruction: v1 (xs, s, m, m-long, l) and v1.5 alike, including the bundled xs model.;; Keep patterns narrow: a false positive is unfixable without a code change, since the;; `ee-embedding-query-prefix` setting can only replace a matched prefix, never suppress it.- {#"(?i)snowflake-arctic-embed" "query: "})+ ;; Hence the leading anchor, so an unrelated model that merely contains the family name is left alone.+ [[#"(?i)(?:^|/)snowflake-arctic-embed-\w+-v[2-9]" "query: "]+ [#"(?i)(?:^|/)snowflake-arctic-embed(?:-|$)" "Represent this sentence for searching relevant passages: "]])(defn- default-query-prefix[model-name]@@ -641,27 +686,10 @@:model-name (semantic-settings/ee-embedding-model):vector-dimensions (semantic-settings/ee-embedding-model-dimensions)})-(defmulti embedding-supported?- "Whether `embedding-model`'s provider is *configured* to compute embeddings — the endpoint/credentials it- needs are present. This is a config-presence check, not a liveness probe: a set URL whose service is down- (or a stopped ollama) still reads as supported and surfaces at call time. Dispatches on provider,- mirroring [[get-embedding]] and the config each provider's impl resolves; a new provider — including a- future in-process embedder — adds a method. The `:default` is false, so an unrecognized provider gates- callers off safely."- {:arglists '([embedding-model])} dispatch-provider)--(defmethod embedding-supported? :default [_] false)--(defmethod embedding-supported? "ai-service" [_]- (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))- (not-empty (llm.settings/ai-service-base-url)))))--(defmethod embedding-supported? "openai" [_]- (boolean (not-empty (semantic-settings/openai-api-key))))--;; ollama's endpoint is hardcoded (localhost:11434) with no setting to check, so config-presence is always-;; true — consistent with ai-service/openai, which likewise check for a configured URL, not a live server.-(defmethod embedding-supported? "ollama" [_] true)+(defn embedding-supported?+ "Whether the selected provider is installed and configured. This is not a remote liveness probe."+ [embedding-model]+ (embeddings.provider/ready? embedding-model))(defn- calc-token-metrics[texts]@@ -704,7 +732,7 @@(comment;; Configuration:- ;; MB_EE_EMBEDDING_PROVIDER: "ai-service" (default), "openai", or "ollama"+ ;; MB_EE_EMBEDDING_PROVIDER: "ai-service" (default), "openai", "ollama", or "in-process";; MB_EE_EMBEDDING_MODEL: optional override (leave empty for provider defaults);; - OpenAI default: "text-embedding-3-small";; - Ollama default: "mxbai-embed-large"diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.cljindex 14f43626796..f14a08791c2 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj@@ -85,8 +85,23 @@index (:index state)table-name (:table-name index)stalled-at (-> state :metadata-row :indexer_stalled_at)- model-mismatch? (not= (:embedding-model index) (semantic.env/get-configured-embedding-model))- embedder-problem (when-not model-mismatch? (embedding-health/embedding-problem))+ ;; The embedding space, not the settings, decides whether the stored vectors are still usable:+ ;; a provider upgrade can resolve unchanged settings to a new space and strand the index.+ ;; Resolution reaches the provider, so it can fail outright (plugin absent, platform+ ;; unsupported) -- that is its own incident, not a mismatch.+ configured (try+ {:space-id (:embedding-space-id+ (semantic.embedding/resolve-model+ (semantic.env/get-configured-embedding-model)))}+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ {:error e}))+ model-mismatch? (and (nil? (:error configured))+ (not= (:embedding-space-id (:embedding-model index))+ (:space-id configured)))+ embedder-problem (when-not (or model-mismatch? (:error configured))+ (embedding-health/embedding-problem))problems (cond-> [](not (active-index-queryable? pgvector table-name))(conj "active index table not queryable")@@ -97,6 +112,10 @@(some? stalled-at)(conj (str "indexer stalled since " stalled-at))+ (some? (:error configured))+ (conj (str "configured embedding model cannot be resolved: "+ (ex-message (:error configured))))+model-mismatch?(conj "active index embedding model does not match configured model")diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 112f22fef3c..ae43d578483 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -27,6 +27,7 @@[toucan2.core :as t2])(:import[com.mchange.v2.c3p0 PooledDataSource]+ [java.nio.charset StandardCharsets][java.time Instant LocalDate OffsetDateTime ZonedDateTime][java.util.concurrent ArrayBlockingQueue RejectedExecutionHandler RejectedExecutionException TimeUnit ThreadPoolExecutor][org.postgresql.util PGobject]))@@ -271,7 +272,7 @@Note: The index parameters will still be available in index_metadata"[identifier]- (if (<= (count identifier) 63)+ (if (<= (alength (.getBytes ^String identifier StandardCharsets/UTF_8)) 63)identifier(str "index_" (buddy-codecs/bytes->hex (buddy-hash/sha1 identifier)))))@@ -285,10 +286,13 @@Table names produced here must stay recognizable by [[index-table-name?]]."[embedding-model]- (let [{:keys [model-name provider vector-dimensions]} embedding-model+ (let [{:keys [model-name provider vector-dimensions embedding-space-id]} embedding-modelprovider-name (embedding/abbrev-provider-name provider)abbrev-model-name (embedding/abbrev-model-name model-name)- ideal-table-name (str "index_" provider-name "_" abbrev-model-name "_" vector-dimensions)]+ space-suffix (when embedding-space-id+ (subs (buddy-codecs/bytes->hex (buddy-hash/sha1 embedding-space-id)) 0 12))+ ideal-table-name (str "index_" provider-name "_" abbrev-model-name "_" vector-dimensions+ (when space-suffix (str "_e" space-suffix)))](hash-identifier-if-exceeds-pg-limit ideal-table-name)))(def ^:private index-table-name-pattern@@ -296,8 +300,9 @@suffix appended by [[metabase-enterprise.semantic-search.pgvector-api/fresh-index]]),[[hash-identifier-if-exceeds-pg-limit]], and the legacy pre-BOT-337 naming era:- index_<provider>_<model>_<dims> e.g. index_ais_text_3_sm_1536- index_<provider>_<model>_<dims>_<digits> force-reset suffix ([[model-table-suffix]])+ index_<provider>_<model>_<dims> legacy name+ index_<provider>_<model>_<dims>_e<12-hex> immutable-space name+ either modern name followed by _<digits> force-reset suffix ([[model-table-suffix]])index_<40-hex-sha1> names exceeding the 63-byte pg identifier limitindex_table_<anything> legacy pre-BOT-337 naming (index_table_<provider>_<model>_<dims>)@@ -306,7 +311,7 @@gives the modern shapes their structure, while the index_table_ prefix — used exclusively by thelegacy era — claims anything under it. Deliberately does NOT match the control-plane tables(index_metadata, index_control, index_gate): no trailing _<digits>, not 40-hex, not index_table_."- #"\Aindex_(?:.+_\d+|[0-9a-f]{40}|table_.+)\z")+ #"\Aindex_(?:.+_\d+(?:_e[0-9a-f]{12})?(?:_\d+)?|[0-9a-f]{40}|table_.+)\z")(defn index-table-name?"Does the bare (schema- and qualifier-stripped) table name look like a semantic-search index table?diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.cljindex d68d7c3cf61..91bd6f8b379 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj@@ -2,7 +2,7 @@"Multi-index management for pgvector semantic search.Tracks multiple embedding model indexes through metadata tables, supporting model switching- without data loss. Each embedding model (provider/name/dimensions) gets its own index table,+ without data loss. Each immutable embedding space gets its own index table,with a control table managing which index is currently active."(:require[honey.sql :as sql]@@ -11,6 +11,7 @@[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.embeddings.provider :as embeddings.provider][metabase.util :as u][metabase.util.log :as log][next.jdbc :as jdbc]@@ -30,7 +31,9 @@[[:id :bigint [:primary-key] [:raw "GENERATED BY DEFAULT AS IDENTITY"]][:provider :text :not-null][:model_name :text :not-null]+ [:model_revision :text :null][:vector_dimensions :int :not-null]+ [:embedding_space_id :text :not-null][:table_name :text :not-null :unique][:index_version :int :not-null][:index_created_at :timestamp-with-time-zone :not-null]@@ -70,14 +73,21 @@[index index-metadata];; if created databases for namespacing in tests we could probably remove this whole idea.(let [{:keys [index-table-qualifier]} index-metadata+ ;; Re-apply the 63-byte hash: the qualifier can push an already-fit name over Postgres's limit,+ ;; and a name silently truncated on CREATE would no longer match table-exists? lookups. The limit+ ;; is per identifier, so hash only the table component -- a schema qualifier (app-db mode) is a+ ;; separate short identifier that must survive, or the table lands outside its isolated schema.qualify (fn [table-name]- (format index-table-qualifier table-name))]+ (let [[schema table] (semantic.util/qualified-table-parts+ (format index-table-qualifier table-name))]+ (cond->> (semantic.index/hash-identifier-if-exceeds-pg-limit table)+ schema (str schema "."))))](-> index(update :table-name qualify))))(def default-index-metadata"The index metadata configuration for a dedicated pgvector database (MB_PGVECTOR_DB_URL)."- {:version "2"+ {:version "3":metadata-table-name "index_metadata":control-table-name "index_control":gate-table-name "index_gate"@@ -92,7 +102,7 @@[[metabase-enterprise.semantic-search.util/table-name-part]].";; HoneySQL renders the dotted keywords as schema-qualified identifiers, quoted or not.(let [schema semantic.db.datasource/app-db-schema]- {:version "2"+ {:version "3":schema schema:metadata-table-name (str schema ".index_metadata"):control-table-name (str schema ".index_control")@@ -222,12 +232,17 @@(defn- row->index [{:keys [providermodel_name+ model_revisionvector_dimensions+ embedding_space_idtable_nameindex_version]}]- {:embedding-model {:provider provider- :model-name model_name- :vector-dimensions vector_dimensions}+ {:embedding-model (cond-> {:provider provider+ :model-name model_name+ :vector-dimensions vector_dimensions+ :embedding-space-id embedding_space_id+ :embedding-spi-version embeddings.provider/embedding-spi-version}+ model_revision (assoc :model-revision model_revision)):table-name table_name:version index_version})@@ -297,7 +312,8 @@{:keys [providermodel-name- vector-dimensions]}+ vector-dimensions+ embedding-space-id]}embedding-modelmodel-rows-sql (-> {:select [:m.* [[:coalesce [:= :c.active_id :m.id] false] :is_active]]@@ -306,7 +322,8 @@:where [:and[:= :provider provider][:= :model_name model-name]- [:= :vector_dimensions vector-dimensions]]+ [:= :vector_dimensions vector-dimensions]+ [:= :embedding_space_id embedding-space-id]]:order-by [[:index_created_at :desc]]}(sql/format :quoted true))model-rows (jdbc/execute! pgvector model-rows-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})@@ -361,13 +378,17 @@{:keys [providermodel-name- vector-dimensions]}+ model-revision+ vector-dimensions+ embedding-space-id]}embedding-modelinsert-sql (-> (sql.helpers/insert-into (keyword metadata-table-name))(sql.helpers/values [{:provider provider:model_name model-name+ :model_revision model-revision:vector_dimensions vector-dimensions+ :embedding_space_id embedding-space-id:table_name index-table-name:index_version index-version:index_created_at [:now]}])diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.cljindex c80c7819e2b..661aecb2c58 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj@@ -11,6 +11,7 @@[metabase-enterprise.semantic-search.db.connection :as semantic.db.connection][metabase-enterprise.semantic-search.db.migration :as semantic.db.migration][metabase-enterprise.semantic-search.dlq :as semantic.dlq]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.gate :as semantic.gate][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]@@ -30,7 +31,6 @@(def pgvector (or @semantic.db/data-source (semantic.db/init-db!)))(def index-metadata semantic.index-metadata/default-index-metadata)- (require '[metabase-enterprise.semantic-search.embedding :as semantic.embedding])(def embedding-model (semantic.embedding/get-configured-model)))(defn- fresh-index [index-metadata embedding-model & {:keys [force-reset?]}]@@ -118,10 +118,11 @@Designed to be called once at application startup (or in tests)."[pgvector index-metadata embedding-model & {:as opts}]- (semantic.db.connection/with-migrate-tx [tx pgvector]- (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata- :embedding-model embedding-model})- (initialize-index! tx index-metadata embedding-model opts)))+ (let [embedding-model (semantic.embedding/resolve-model embedding-model)]+ (semantic.db.connection/with-migrate-tx [tx pgvector]+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata+ :embedding-model embedding-model})+ (initialize-index! tx index-metadata embedding-model opts))));; query/index-mgmt require an active index to be established first.;; init-semantic-search! must be called on startupdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex cf6970faf98..073fd6c0a91 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.semantic-search.settings(:require+ [clojure.string :as str][metabase.events.core :as events][metabase.llm.settings :as llm-settings][metabase.search.config :as search.config]@@ -10,12 +11,8 @@;; here, not there, so it's valid wherever the setter runs regardless of handler-namespace load order.(events/derive! :event/semantic-search-hnsw-enabled :metabase/event)-(def ^:private valid-embedding-providers- "The set of valid embedding provider names."- #{"ai-service" "openai" "ollama"})-(defsetting ee-embedding-provider- (deferred-tru "The embedding provider to use (`openai`, `ollama`, or `ai-service`)")+ (deferred-tru "The registered embedding provider to use"):encryption :no:visibility :settings-manager:default "ai-service"@@ -23,11 +20,12 @@:export? false:doc false:setter (fn [new-value]- (when (and new-value (not (contains? valid-embedding-providers new-value)))- (throw (ex-info (str "Invalid embedding provider: " (pr-str new-value)- ". Valid providers are: " (pr-str valid-embedding-providers))- {:invalid-value new-value- :valid-values valid-embedding-providers})))+ ;; Provider plugins may be configured before they are loaded. Readiness reports an unregistered+ ;; provider as unavailable, so validation here only needs to protect the registry key contract.+ (when-not (or (nil? new-value)+ (and (string? new-value) (not (str/blank? new-value))))+ (throw (ex-info "Embedding provider must be a non-blank string."+ {:invalid-value new-value})))(setting/set-value-of-type! :string :ee-embedding-provider new-value)))(defsetting ee-embedding-model@@ -41,8 +39,9 @@(defsetting ee-embedding-query-prefix(deferred-tru(str "Prefix prepended to search queries (but not indexed documents) before embedding them, as expected by "- "asymmetric retrieval models such as the snowflake-arctic-embed family (`query: `). It is prepended "- "verbatim, so include any trailing separator. Leave empty to use the default for the model family."))+ "asymmetric retrieval models such as the snowflake-arctic-embed family. It is prepended verbatim, so "+ "include any trailing separator. Leave empty to use the default for the model, which varies by "+ "generation: Arctic Embed v2.0 expects `query: `, earlier versions expect a longer instruction.")):encryption :no:visibility :settings-manager:default nildiff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.cljindex f72c3f0473b..5252fbe790e 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/cli_test.clj@@ -11,6 +11,7 @@[metabase-enterprise.data-complexity-score.synonym-source :as synonym-source][metabase-enterprise.data-complexity-score.task.complexity-score :as task.complexity-score][metabase.app-db.core :as mdb]+ [metabase.embeddings.startup :as embeddings.startup][metabase.test :as mt][metabase.util :as u][metabase.util.json :as json]))@@ -387,6 +388,22 @@(#'cli/run-cli {:source "appdb" :write-to-appdb false})(is (false? @persisted?))))))+(deftest ^:sequential run-cli-appdb-mode-registers-in-process-provider-test+ (testing "appdb mode registers the in-process embedding provider before it scores"+ ;; The standalone CLI skips `metabase.core.core/init!`, where the server registers the provider. Without+ ;; this a synonym axis configured for `in-process` fails with `:provider-not-registered`.+ (let [order (atom [])]+ (mt/with-dynamic-fn-redefs [mdb/setup-db-without-migrations! (fn [])+ embeddings.startup/ensure-in-process-provider! (fn [] (swap! order conj :register))+ complexity/complexity-scores (fn [& _]+ (swap! order conj :score)+ {:meta {}})+ synonym-source/complexity-scores-opts (constantly {})+ metabot-scope/internal-metabot-scope (constantly {})+ data-complexity-score/record-score! (fn [& _])]+ (#'cli/run-cli {:source "appdb" :write-to-appdb false})+ (is (= [:register :score] @order))))))+(deftest ^:parallel dir-digest-is-stable-and-content-sensitive-test(testing "dir-digest produces the same value for the same content"(let [d1 (#'representation/dir-digest representation-fixture-dir)diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex 494c0287436..4ecd37baa6b 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -23,6 +23,7 @@[metabase.audit-app.core :as audit][metabase.collections.core :as collections][metabase.collections.test-utils :as collections.tu]+ [metabase.embeddings.provider :as embeddings.provider][metabase.task.core :as task][metabase.test :as mt][toucan2.core :as t2]))@@ -31,9 +32,18 @@(def ^:private test-entity-ids (atom 0))-(defmethod semantic-search/get-embeddings-batch ::dimension-translation-test- [embedding-model _texts & _opts]- [embedding-model])+(def ^:private captured-translation-model+ "Descriptor the provider layer receives, captured by the dimension-translation test." (atom nil))++;; A capturing provider so the dimension-translation test can inspect the descriptor the client hands down.+(embeddings.provider/register-provider!+ "dimension-translation-test"+ {:embedding-spi-version embeddings.provider/embedding-spi-version+ :readiness (constantly {:ready? true})+ :resolve-model embeddings.provider/legacy-resolved-model+ :embed-texts (fn [model texts _opts]+ (reset! captured-translation-model model)+ (mapv (fn [_] (vec (repeat (:vector-dimensions model) 0.0))) texts))})(defn- entity"Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`@@ -734,6 +744,24 @@(is (fn? embedder)"synonym-source returns a fresh provider-embedder for the descriptor")))))+(deftest ^:sequential synonym-source-in-process-opts-pin-bundled-minilm-test+ (testing "the in-process synonym provider selects the bundled MiniLM model, not Library retrieval's Arctic model"+ (test-util/with-synonym-source [:provider "in-process"]+ (let [{:keys [embedder embedding-model-meta text-variant]} (synonym-source/complexity-scores-opts)+ bundled-minilm {:provider "in-process"+ :model-name "sentence-transformers/all-MiniLM-L6-v2"+ :model-dimensions 384}+ captured (atom nil)]+ (is (= bundled-minilm embedding-model-meta))+ (is (= :names-split text-variant))+ (mt/with-dynamic-fn-redefs [embeddings/get-embeddings-batch+ (fn [model texts & _opts]+ (reset! captured model)+ (repeat (count texts) [1.0]))]+ (embedder [{:id 1 :name "orders" :kind :table}]))+ (is (= bundled-minilm @captured)+ "the embedder asks the provider SPI for the bundled MiniLM model")))))+(deftest ^:sequential provider-embedder-suppresses-token-tracking-test(testing "provider-embedder always passes :record-tokens? false to get-embeddings-batch";; Complexity scoring isn't user-driven search traffic; the score itself is the analytics@@ -749,11 +777,10 @@(is (false? (:record-tokens? @captured)))))))(deftest ^:parallel embeddings-client-translates-neutral-dimension-key-test- (let [[translated]- (embeddings/get-embeddings-batch- {:provider ::dimension-translation-test, :model-name "fake", :model-dimensions 384}- ["orders"])]- (is (= 384 (:vector-dimensions translated)))))+ (embeddings/get-embeddings-batch+ {:provider "dimension-translation-test", :model-name "fake", :model-dimensions 384}+ ["orders"])+ (is (= 384 (:vector-dimensions @captured-translation-model))))(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test(testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"diff --git a/enterprise/backend/test/metabase_enterprise/embeddings/client_test.clj b/enterprise/backend/test/metabase_enterprise/embeddings/client_test.cljnew file mode 100644index 00000000000..a21091c6678--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/embeddings/client_test.clj@@ -0,0 +1,33 @@+(ns metabase-enterprise.embeddings.client-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.embeddings.client :as embeddings.client]+ [metabase-enterprise.semantic-search.core :as semantic-search]))++(set! *warn-on-reflection* true)++(deftest legacy-model-dimensions-are-normalized-test+ (let [captured (atom nil)]+ (with-redefs [semantic-search/get-embeddings-batch+ (fn [model texts & {:as opts}]+ (reset! captured [model texts opts])+ [[1.0 2.0]])]+ (is (= [[1.0 2.0]]+ (embeddings.client/get-embeddings-batch+ {:provider "ai-service" :model-name "model" :model-dimensions 2}+ ["text"]+ :record-tokens? false)))+ (is (= [{:provider "ai-service" :model-name "model" :vector-dimensions 2}+ ["text"]+ {:record-tokens? false}]+ @captured)))))++(deftest conflicting-dimension-keys-are-rejected-test+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"conflicting dimension values"+ (embeddings.client/get-embeddings-batch+ {:provider "ai-service"+ :model-name "model"+ :model-dimensions 2+ :vector-dimensions 3}+ ["text"]))))diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.cljindex 6966c4046d1..da427d0695f 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj@@ -46,6 +46,85 @@(is (> (:total_score (score {:distance 0.1 :doc_type "name"}))(:total_score (score {:distance 0.5 :doc_type "name"})))))))+(deftest search-uses-reconcile-lock-test+ (let [calls (atom [])+ lock-count (atom 0)]+ (mt/with-premium-features #{:library-retrieval}+ (with-redefs [entity-retrieval.core/available? (constantly true)+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ reconcile/with-index-read-lock (fn [ds f]+ (let [n (swap! lock-count inc)+ conn (keyword (str "locked-connection-" n))]+ (swap! calls conj [:lock ds])+ (f conn)))+ index-table/index-status (fn [conn _model]+ (swap! calls conj [:compatible conn])+ :compatible)+ semantic.embedding/get-embedding (fn [_model text & _opts]+ (swap! calls conj [:embed text])+ [0.0 0.0 0.0 0.0])+ jdbc/execute! (fn [conn & _args]+ (swap! calls conj [:query conn])+ [])]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10)))+ (is (= [[:lock ::datasource]+ [:compatible :locked-connection-1]+ [:embed "the query"]+ [:lock ::datasource]+ [:compatible :locked-connection-2]+ [:query :locked-connection-2]]+ @calls)+ "both preflight and query compatibility checks happen behind non-blocking reconcile locks")))))++(deftest search-skips-embedding-for-an-incompatible-index-test+ (mt/with-premium-features #{:library-retrieval}+ (with-redefs [entity-retrieval.core/available? (constantly true)+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ index-table/index-status (constantly :incompatible)+ semantic.embedding/get-embedding (fn [& _]+ (throw (ex-info "must not embed" {})))+ reconcile/with-index-read-lock (fn [_ds f]+ (f ::locked-connection))]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))++(deftest search-degrades-when-reconcile-holds-the-lock-test+ (mt/with-premium-features #{:library-retrieval}+ (with-redefs [entity-retrieval.core/available? (constantly true)+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ index-table/index-status (fn [& _]+ (throw (ex-info "must not touch index metadata" {})))+ semantic.embedding/get-embedding (fn [& _]+ (throw (ex-info "must not embed" {})))+ reconcile/with-index-read-lock (constantly nil)+ jdbc/execute! (fn [& _]+ (throw (ex-info "must not query" {})))]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))++(deftest search-degrades-when-configured-model-cannot-be-resolved-test+ (mt/with-premium-features #{:library-retrieval}+ (with-redefs [entity-retrieval.core/available? (constantly true)+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ semantic.embedding/resolve-model (fn [_]+ (throw (ex-info "model changed"+ {:type ::model-changed})))+ index-table/index-status (fn [& _]+ (throw (ex-info "must not inspect index" {})))+ semantic.embedding/get-embedding (fn [& _]+ (throw (ex-info "must not embed" {})))+ reconcile/with-index-read-lock (fn [& _]+ (throw (ex-info "must not lock" {})))+ jdbc/execute! (fn [& _]+ (throw (ex-info "must not query" {})))]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))+(deftest dispatch-without-pgvector-test(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully";; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:@@ -302,15 +381,18 @@(testing "search ranks library documents by cosine similarity, nearest first"(when semantic.db.datasource/db-url(mt/with-premium-features #{:library :library-retrieval}- (let [suffix (System/nanoTime)- ds (semantic.db.datasource/ensure-initialized-data-source!)- q "the query"]+ (let [suffix (System/nanoTime)+ ds (semantic.db.datasource/ensure-initialized-data-source!)+ q "the query"+ prefixed-q "query: the query"+ model (semantic.tu/resolved-mock-embedding-model+ :model-name "Snowflake/snowflake-arctic-embed-l-v2.0")](mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model- (constantly semantic.tu/mock-embedding-model)]+ (constantly model)](binding [index-table/*tables* {:vectors (str "library_entity_index_test_" suffix):meta (str "library_entity_index_meta_test_" suffix)}]- ;; the two tables' name docs embed "near"/"far"; q ties "near" exactly and is orthogonal to "far".- (semantic.tu/with-mock-embeddings {q [1.0 0.0 0.0 0.0]+ ;; Arctic's prefixed query ties "near" exactly and is orthogonal to "far".+ (semantic.tu/with-mock-embeddings {prefixed-q [1.0 0.0 0.0 0.0]"near" [1.0 0.0 0.0 0.0]"far" [0.0 1.0 0.0 0.0]}(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}@@ -321,7 +403,7 @@:model/Table {far-id :id} {:db_id db-id :collection_id data-id :is_published true:active true :name "far" :display_name "far"}](try- (reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model))+ (reconcile/reconcile! ds (constantly model))(testing "the nearer table ranks first; each hit carries the entity ref + matched doc"(is (=? [{:entity {:model "table" :id near-id} :doc_type "name" :doc_text "near"}{:entity {:model "table" :id far-id}}]@@ -456,8 +538,21 @@(is (=? [{:model "table" :id table-id}](distinct (map :entity (entity-retrieval.core/search-unfiltered "orders" 10))))"the query really runs, so the [] below is the degrade path and not a fallback")- ;; a dim-1 query vector against the dim-4 index column -> pgvector SQLState 22000- (is (= [] (entity-retrieval.core/search-unfiltered "q" 10))))+ ;; Shrink the configured model to one dimension, the way an upgrade would, keeping it+ ;; self-consistent so the provider resolves it instead of rejecting a changed space.+ ;; The meta row still describes the dim-4 build, so the compatibility preflight would+ ;; answer [] on its own; stub it aside to reach the case this test is about — a dim-1+ ;; literal meeting the vector(4) column, which is pgvector SQLState 22000.+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly (semantic.tu/resolved-mock-embedding-model+ :vector-dimensions 1))+ index-table/index-status (constantly :compatible)]+ (mt/with-log-messages-for-level [messages [metabase-enterprise.entity-retrieval.core :warn]]+ (is (= [] (entity-retrieval.core/search-unfiltered "q" 10)))+ (is (=? [{:level :warn+ :message #"(?s)library entity index incompatible with the query vector.*"}]+ (messages))+ "the query reached pgvector and degraded on 22000, not on an earlier preflight"))))(finally(jdbc/execute! ds [(str "DROP TABLE IF EXISTS \"" (index-table/vectors-table)"\", \"" (index-table/meta-table) "\"")])))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.cljindex a15415a005c..56ffee646fb 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj@@ -60,75 +60,74 @@(is (= ["CREATE SCHEMA IF NOT EXISTS \"library_retrieval\""] @statements)"nothing else creates it: semantic search provisions its own, and we run without semantic search")))-(deftest legacy-tables-are-moved-not-rebuilt-test+(deftest incompatible-legacy-tables-are-dropped-test(let [statements (atom [])- exists (atom #{"library_entity_index" "library_entity_index_meta"})]- (with-redefs-fn {#'index-table/table-exists? (fn [_ table] (contains? @exists table))- #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)- #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}+ located (atom {"library_entity_index" {:schema-name "legacy.with.dot"+ :table-name "library_entity_index"}+ "library_entity_index_meta" {:schema-name "legacy.with.dot"+ :table-name "library_entity_index_meta"}})]+ (with-redefs-fn {#'index-table/legacy-table-in-search-path (fn [_ table] (get @located table))+ #'jdbc/execute! (fn [_ sql] (swap! statements conj (first sql)) nil)+ #'semantic.db.datasource/db-url "jdbc:postgresql://stub"}(fn []- (testing "an index built before the schema existed is moved, keeping its rows"- (#'index-table/adopt-legacy-tables! ::tx)- (is (= ["ALTER TABLE \"library_entity_index\" SET SCHEMA \"library_retrieval\""- "ALTER TABLE \"library_entity_index_meta\" SET SCHEMA \"library_retrieval\""]+ (testing "an index built before immutable space identity is discarded"+ (#'index-table/drop-legacy-tables! ::tx)+ (is (= ["DROP TABLE \"legacy.with.dot\".\"library_entity_index\""+ "DROP TABLE \"legacy.with.dot\".\"library_entity_index_meta\""]@statements)))- (testing "and left alone once moved"+ (testing "there is nothing to do after the legacy tables are gone"(reset! statements [])- (reset! exists #{"library_retrieval.library_entity_index"- "library_retrieval.library_entity_index_meta"})- (#'index-table/adopt-legacy-tables! ::tx)+ (reset! located {})+ (#'index-table/drop-legacy-tables! ::tx)(is (empty? @statements)))- (testing "a qualified table standing beside a leftover legacy one keeps its rows"+ (testing "same-named app db tables are never dropped"(reset! statements [])- (reset! exists #{"library_entity_index"- "library_entity_index_meta"- "library_retrieval.library_entity_index"- "library_retrieval.library_entity_index_meta"})- (#'index-table/adopt-legacy-tables! ::tx)- (is (empty? @statements)- "SET SCHEMA onto an occupied name errors, aborting the whole ensure-tables! transaction"))- (testing "an app db is never adopted from: nothing there can be an old index of ours"- (reset! statements [])- (reset! exists #{"library_entity_index" "library_entity_index_meta"})+ (reset! located {"library_entity_index" {:schema-name "public"+ :table-name "library_entity_index"}+ "library_entity_index_meta" {:schema-name "public"+ :table-name "library_entity_index_meta"}})(with-redefs [semantic.db.datasource/db-url nil]- (#'index-table/adopt-legacy-tables! ::tx))+ (#'index-table/drop-legacy-tables! ::tx))(is (empty? @statements)- "an application table on the search path answers to the bare name, and must not be moved"))))))+ "application tables with the legacy names do not belong to the library index"))))))-(deftest ^:synchronized legacy-tables-are-adopted-in-a-dedicated-store-test- (testing "a real SET SCHEMA carries the rows an upgrading dedicated store already embedded"+(deftest ^:synchronized incompatible-legacy-tables-are-replaced-in-a-dedicated-store-test+ (testing "old vectors without immutable space identity are replaced with current empty tables"(when (semantic.db.datasource/dedicated-url-configured?)- ;; its own database: adoption only runs under the real table names, which a shared store may hold+ ;; its own database: cleanup only runs under the real table names, which a shared store may hold(semantic.tu/with-test-db! {:dbname "library_retrieval_adoption_test" :mode :blank :cleanup :both}(let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)](jdbc/execute! pgvector ["CREATE EXTENSION IF NOT EXISTS vector"])- ;; both bare names the index carried before it had a schema of its own, with a meta row for the- ;; model it was built against -- an upgrade the reconciler should find nothing to redo- (jdbc/execute! pgvector ["CREATE TABLE library_entity_index- (doc_id text primary key, entity_type text, entity_local_id bigint,- doc_type text, doc_text text, doc_embedding vector(4))"])- (jdbc/execute! pgvector ["INSERT INTO library_entity_index- VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])- (jdbc/execute! pgvector ["CREATE TABLE library_entity_index_meta- (id smallint primary key, provider text, model_name text,- vector_dimensions int, schema_version int,- updated_at timestamptz, reconciled_at timestamptz)"])- (jdbc/execute! pgvector ["INSERT INTO library_entity_index_meta- VALUES (1, ?, ?, ?, ?, now(), now())"- (:provider semantic.tu/mock-embedding-model)- (:model-name semantic.tu/mock-embedding-model)- (:vector-dimensions semantic.tu/mock-embedding-model)- index-table/schema-version])- (is (= :ok (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model))- "the adopted meta row still describes the configured model, so nothing is rebuilt")- (is (= ["legacy-doc"]- (map :doc_id (jdbc/execute! pgvector- [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]- {:builder-fn jdbc.rs/as-unqualified-lower-maps})))- "moved, not recreated empty -- otherwise the whole library re-embeds on upgrade")- (is (empty? (jdbc/execute! pgvector [(str "SELECT tablename FROM pg_tables WHERE schemaname = 'public'"- " AND tablename LIKE 'library_entity_index%'")]))- "moved, not copied -- a leftover is a stale duplicate of the whole index"))))))+ (jdbc/with-transaction [tx pgvector]+ (jdbc/execute! tx ["CREATE SCHEMA \"legacy.retrieval\""])+ (jdbc/execute! tx ["SET LOCAL search_path TO \"legacy.retrieval\", public"])+ ;; both bare names and the metadata shape the index carried before immutable space identity+ (jdbc/execute! tx ["CREATE TABLE library_entity_index+ (doc_id text primary key, entity_type text, entity_local_id bigint,+ doc_type text, doc_text text, doc_embedding vector(4))"])+ (jdbc/execute! tx ["INSERT INTO library_entity_index+ VALUES ('legacy-doc', 'table', 1, 'name', 'legacy', '[0,0,0,0]')"])+ (jdbc/execute! tx ["CREATE TABLE library_entity_index_meta+ (id smallint primary key, provider text, model_name text,+ vector_dimensions int, schema_version int,+ updated_at timestamptz, reconciled_at timestamptz)"])+ (jdbc/execute! tx ["INSERT INTO library_entity_index_meta+ VALUES (1, ?, ?, ?, ?, now(), now())"+ (:provider semantic.tu/mock-embedding-model)+ (:model-name semantic.tu/mock-embedding-model)+ (:vector-dimensions semantic.tu/mock-embedding-model)+ index-table/schema-version])+ (#'index-table/drop-legacy-tables! tx))+ (is (= :created (index-table/ensure-tables! pgvector semantic.tu/mock-embedding-model)))+ (is (empty? (jdbc/execute! pgvector+ [(format "SELECT doc_id FROM %s" (index-table/vectors-table-sql))]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))+ "unknown legacy vectors are not relabeled as the current embedding space")+ (is (empty? (jdbc/execute! pgvector+ [(str "SELECT schemaname, tablename FROM pg_tables"+ " WHERE schemaname <> 'library_retrieval'"+ " AND tablename LIKE 'library_entity_index%'")]))+ "the incompatible tables do not remain in their custom search-path schema"))))))(deftest reconcile-watermark-precedes-appdb-read-test(let [events (atom [])]diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.cljindex e1693850d91..c4dd5ca27b6 100644--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj@@ -13,7 +13,9 @@[metabase.test.fixtures :as fixtures][next.jdbc :as jdbc][next.jdbc.result-set :as jdbc.rs]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.sql Connection)))(set! *warn-on-reflection* true)@@ -93,6 +95,30 @@(index-table/vectors-table) ", "(index-table/meta-table))])))))))+(deftest ^:sequential with-index-read-lock-contention-integration-test+ (when semantic.db.datasource/db-url+ (let [ds (semantic.db.datasource/ensure-initialized-data-source!)+ lock-id (var-get #'reconcile/reconcile-lock-id)+ calls (atom 0)+ callback (fn [_]+ (swap! calls inc)+ ::called)]+ (with-open [^Connection writer (jdbc/get-connection ds)]+ (let [acquired? (:acquired+ (jdbc/execute-one!+ writer+ [(format "SELECT pg_try_advisory_lock(%d) AS acquired" lock-id)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))]+ (is acquired? "the exclusive test lock was acquired without blocking")+ (when acquired?+ (try+ (is (nil? (reconcile/with-index-read-lock ds callback)))+ (is (zero? @calls) "the read callback is skipped while reconcile owns the lock")+ (finally+ (jdbc/execute! writer [(format "SELECT pg_advisory_unlock(%d)" lock-id)])))+ (is (= ::called (reconcile/with-index-read-lock ds callback)))+ (is (= 1 @calls) "the callback runs after the exclusive lock is released")))))))+(defn- index-rows [ds](jdbc/execute! ds[(format "SELECT doc_id, entity_type, entity_local_id, doc_type, doc_text FROM \"%s\""@@ -201,7 +227,7 @@(is (seq (docs-for ds "table" table-id)))(is (some? (reconciled-at! ds)) "a converged reconcile stamps reconciled_at"))(testing "a model-identity change drops the vectors table and the next run re-embeds everything"- (let [new-model (assoc model :model-name "model-v2")]+ (let [new-model (semantic.tu/resolved-mock-embedding-model :model-name "model-v2")](is (= :rebuilt (index-table/ensure-tables! ds new-model)))(is (= [] (index-rows ds)))(is (nil? (reconciled-at! ds))@@ -216,6 +242,72 @@(testing "the rebuild heals the meta row, so it doesn't recur on the next sync"(is (= :ok (index-table/ensure-tables! ds new-model)))))))))))+(deftest ^:sequential embedding-space-change-rebuilds-test+ (with-isolated-index [ds]+ (let [model semantic.tu/mock-embedding-model+ changed-space (update model :embedding-space-id str "-changed")]+ (is (= :created (index-table/ensure-tables! ds model)))+ (jdbc/execute! ds [(format (str "INSERT INTO \"%s\" "+ "(doc_id, entity_type, entity_local_id, doc_type, doc_text, doc_embedding) "+ "VALUES ('sentinel', 'table', 1, 'name', 'sentinel', '[0,0,0,0]')")+ (index-table/vectors-table))])+ (testing "the same provider/name/dimensions with a different immutable space is incompatible"+ (is (= :incompatible (index-table/index-status ds changed-space)))+ (is (= :rebuilt (index-table/ensure-tables! ds changed-space)))+ (is (empty? (index-rows ds))))+ (testing "the rebuilt identity is stable"+ (is (= :compatible (index-table/index-status ds changed-space)))+ (is (= :ok (index-table/ensure-tables! ds changed-space)))))))++(deftest ^:sequential version-1-meta-upgrade-rebuilds-test+ (with-isolated-index [ds]+ (let [model semantic.tu/mock-embedding-model]+ (jdbc/execute! ds [(format (str "CREATE TABLE \"%s\" ("+ "id smallint PRIMARY KEY, provider text NOT NULL, model_name text NOT NULL, "+ "vector_dimensions int NOT NULL, schema_version int NOT NULL, "+ "updated_at timestamptz NOT NULL)")+ (index-table/meta-table))])+ (jdbc/execute! ds [(format (str "INSERT INTO \"%s\" "+ "(id, provider, model_name, vector_dimensions, schema_version, updated_at) "+ "VALUES (1, 'mock', 'model', 4, 1, NOW())")+ (index-table/meta-table))])+ (jdbc/execute! ds [(format "CREATE TABLE \"%s\" (sentinel int)" (index-table/vectors-table))])+ (jdbc/execute! ds [(format "INSERT INTO \"%s\" VALUES (1)" (index-table/vectors-table))])+ (testing "a legacy table is rebuilt once rather than relabeled as the resolved space"+ (is (= :rebuilt (index-table/ensure-tables! ds model)))+ (is (empty? (index-rows ds))))+ (let [meta-row (jdbc/execute-one! ds+ [(format (str "SELECT embedding_space_id, schema_version "+ "FROM \"%s\" WHERE id = 1")+ (index-table/meta-table))]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ column-row (jdbc/execute-one! ds+ ["SELECT is_nullable FROM information_schema.columns WHERE table_name = ? AND column_name = 'embedding_space_id'"+ (index-table/meta-table)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (is (= {:embedding_space_id (:embedding-space-id model)+ :schema_version index-table/schema-version}+ meta-row))+ (is (= "NO" (:is_nullable column-row))))+ (let [execute! jdbc/execute!+ statements (atom [])]+ (with-redefs [jdbc/execute! (fn [connectable sql-params & opts]+ (swap! statements conj (first sql-params))+ (apply execute! connectable sql-params opts))]+ (is (= :ok (index-table/ensure-tables! ds model))))+ (is (not-any? #(re-find #"ALTER COLUMN embedding_space_id SET NOT NULL" (str %)) @statements)+ "steady-state reconcile does not reacquire an ACCESS EXCLUSIVE lock"))+ (testing "a manually nullable metadata column is healed once"+ (jdbc/execute! ds [(format "ALTER TABLE \"%s\" ALTER COLUMN embedding_space_id DROP NOT NULL"+ (index-table/meta-table))])+ (is (= :ok (index-table/ensure-tables! ds model)))+ (is (= "NO" (:is_nullable+ (jdbc/execute-one!+ ds+ ["SELECT is_nullable FROM information_schema.columns WHERE table_name = ? AND column_name = 'embedding_space_id'"+ (index-table/meta-table)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))))))+(deftest ^:sequential measures-and-segments-indexed-and-hydrated-test(testing "measures/segments on a published library table are indexed and hydrate with parent-table context"(mt/with-premium-features #{:library :library-retrieval}@@ -427,7 +519,7 @@(is (seq (docs-for ds "table" b-id)));; a targeted reconcile of A under a new model identity forces ensure-tables! to rebuild (empty);;; it must repopulate B too, not leave it missing until the periodic backstop.- (let [new-model (assoc model :model-name "model-v2")]+ (let [new-model (semantic.tu/resolved-mock-embedding-model :model-name "model-v2")](is (:rebuilt? (reconcile/reconcile-entity! ds (constantly new-model) "table" a-id)))(is (seq (docs-for ds "table" a-id)) "A repopulated after the rebuild")(is (seq (docs-for ds "table" b-id)) "B repopulated too (not dropped by the rebuild)")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.cljindex 9a81ec992f1..e6a0c155597 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/appdb_pgvector_mode_test.clj@@ -10,13 +10,17 @@(:require[clojure.string :as str][clojure.test :refer [deftest is testing use-fixtures]]+ [metabase-enterprise.semantic-search.core :as semantic.core][metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase.app-db.core :as mdb]+ [metabase.embeddings.provider :as embeddings.provider]+ [metabase.embeddings.startup :as embeddings.startup][metabase.test :as mt][metabase.test.fixtures :as fixtures][next.jdbc :as jdbc]@@ -67,6 +71,15 @@[](Boolean/parseBoolean (System/getenv "MB_APPDB_PGVECTOR_MODE_TEST")))+(defn- in-process-embedder-opted-in?+ []+ (Boolean/parseBoolean (System/getenv "MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST")))++(def ^:private in-process-embedding-model+ {:provider "in-process"+ :model-name "Snowflake/snowflake-arctic-embed-xs"+ :vector-dimensions 384})+(deftest ^:synchronized appdb-pgvector-round-trip-test(cond(not (opted-in?))@@ -118,7 +131,7 @@(is (contains? tables "index_metadata"))(is (contains? tables "index_control"))(is (contains? tables "index_gate"))- (is (contains? tables "index_mock_model_4"))+ (is (contains? tables (semantic.index/model-table-name semantic.tu/mock-embedding-model)))(is (some #(str/starts-with? % "dlq_") tables))))(testing "search round-trips through the app-db pool";; results are reconstructed from legacy_input, so the mock card comes back as model+id@@ -133,3 +146,54 @@(is (= public-before (tables-in-schema app-db "public")))))(finally(jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"])))))))))++(deftest ^:synchronized appdb-pgvector-in-process-embedder-round-trip-test+ (cond+ (not (in-process-embedder-opted-in?))+ (testing "real in-process inference requires its dedicated CI opt-in — skipping"+ (is true))++ (not (opted-in?))+ (testing "the in-process round trip requires the app-db pgvector opt-in"+ (is false "MB_IN_PROCESS_EMBEDDER_SEMANTIC_SEARCH_TEST requires MB_APPDB_PGVECTOR_MODE_TEST"))++ (not (appdb-can-host-pgvector?))+ (testing "opted in, but the app db can't host pgvector"+ (is false "the in-process round trip expects a pgvector-capable Postgres app db"))++ (schema-exists? (mdb/data-source) "semantic_search")+ (testing "opted in, but a semantic_search schema already exists"+ (is false "the appdb-mode job should start from a fresh app db with no semantic_search schema"))++ :else+ (mt/with-premium-features #{:semantic-search}+ (embeddings.startup/ensure-in-process-provider!)+ (is (embeddings.provider/registered? "in-process"))+ (is (semantic.embedding/embedding-supported? in-process-embedding-model))+ (with-redefs [semantic.db.datasource/db-url nil+ semantic.db.datasource/data-source (atom nil)+ semantic.db.datasource/app-db-pgvector-support (atom nil)+ semantic.db.datasource/probe-cooldown-timer (atom nil)+ semantic.db.datasource/logged-pgvector-absent? (atom false)+ semantic.embedding/get-configured-model (constantly in-process-embedding-model)]+ (let [app-db (mdb/data-source)]+ (try+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)+ documents (mapv #(assoc % :archived false) (semantic.tu/mock-documents))]+ (is (semantic.core/supported?) "the semantic search engine accepts the registered provider")+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata+ (semantic.env/get-configured-embedding-model))+ (semantic.pgvector-api/index-documents! pgvector index-metadata documents)+ (testing "real in-process embeddings flow through indexing and semantic querying"+ (is (= {:model "card" :id 123}+ (-> (semantic.tu/with-weights {:semantic-distance 1}+ (mt/with-test-user :crowberto+ (semantic.pgvector-api/query pgvector index-metadata+ {:search-string "puppy"+ :vector-search-strategy :brute-force})))+ :results+ first+ (select-keys [:model :id]))))))+ (finally+ (jdbc/execute! app-db ["DROP SCHEMA IF EXISTS semantic_search CASCADE"]))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.cljindex c1b44b1b897..7e308388f15 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj@@ -9,11 +9,13 @@[metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.indexer :as semantic.indexer]+ [metabase-enterprise.semantic-search.models.token-tracking :as semantic.models.token-tracking][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase.analytics-interface.core :as analytics][metabase.analytics.snowplow-test :as snowplow-test]+ [metabase.embeddings.provider :as embeddings.provider][metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features][metabase.test :as mt]@@ -52,7 +54,17 @@(is (= {:provider "openai":model-name "text-embedding-3-small":vector-dimensions 1536}- (embedding/get-configured-model))))))+ (embedding/get-configured-model))))+ (mt/with-temporary-setting-values [ee-embedding-provider "in-process"+ ee-embedding-model "Snowflake/snowflake-arctic-embed-l-v2.0"+ ee-embedding-model-dimensions 1024]+ (is (= {:provider "in-process"+ :model-name "Snowflake/snowflake-arctic-embed-l-v2.0"+ :vector-dimensions 1024}+ (embedding/get-configured-model))))+ (testing "plugin-defined provider names do not require a core allowlist change"+ (mt/with-temporary-setting-values [ee-embedding-provider "openrouter-plugin"]+ (is (= "openrouter-plugin" (:provider (embedding/get-configured-model))))))))(deftest test-model-dimensions-with-settings(testing "model-dimensions uses setting defaults when override is nil"@@ -62,18 +74,39 @@(mt/with-temporary-setting-values [ee-embedding-model-dimensions 768](is (= 768 (:vector-dimensions (embedding/get-configured-model)))))))+(deftest embedding-provider-setting-validation-test+ (doseq [invalid-value [42 [] {} " "]]+ (testing (str "rejects " (pr-str invalid-value))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"must be a non-blank string"+ (semantic.settings/ee-embedding-provider! invalid-value))))))+(deftest prefix-search-query-test- (let [arctic {:provider "ai-service" :model-name "Snowflake/snowflake-arctic-embed-l-v2.0" :vector-dimensions 1024}- openai {:provider "openai" :model-name "text-embedding-3-small" :vector-dimensions 1536}]- (mt/with-temporary-setting-values [ee-embedding-query-prefix nil]- (testing "arctic-family models get the `query: ` prefix by default"- (is (= "query: hello" (embedding/prefix-search-query arctic "hello"))))- (testing "models without a known family default are left unprefixed"- (is (= "hello" (embedding/prefix-search-query openai "hello")))))- (testing "the setting overrides the model-family default, verbatim"- (mt/with-temporary-setting-values [ee-embedding-query-prefix "search_query: "]- (is (= "search_query: hello" (embedding/prefix-search-query arctic "hello")))- (is (= "search_query: hello" (embedding/prefix-search-query openai "hello")))))))+ (let [cqe "Represent this sentence for searching relevant passages: "+ ->model (fn [model-name] {:provider "ai-service" :model-name model-name :vector-dimensions 1024})+ prefix-all (fn [model-names]+ (into {}+ (map (juxt identity #(embedding/prefix-search-query (->model %) "hello")))+ model-names))]+ (testing "the default prefix follows the Arctic Embed generation, which shortened it at v2.0"+ (let [expected {"Snowflake/snowflake-arctic-embed-xs" (str cqe "hello")+ "Snowflake/snowflake-arctic-embed-l" (str cqe "hello")+ "Snowflake/snowflake-arctic-embed-m-long" (str cqe "hello")+ "Snowflake/snowflake-arctic-embed-m-v1.5" (str cqe "hello")+ "Snowflake/snowflake-arctic-embed-l-v2.0" "query: hello"+ "Snowflake/snowflake-arctic-embed-m-v2.0" "query: hello"+ "text-embedding-3-small" "hello"+ ;; merely containing the family name is not enough: the prefix would be unsuppressable+ "acme/my-snowflake-arctic-embed-clone" "hello"}]+ (mt/with-temporary-setting-values [ee-embedding-query-prefix nil]+ (is (= expected (prefix-all (keys expected)))))))+ (testing "the setting overrides every model-family default, verbatim"+ (let [model-names ["Snowflake/snowflake-arctic-embed-xs"+ "Snowflake/snowflake-arctic-embed-l-v2.0"+ "text-embedding-3-small"]]+ (mt/with-temporary-setting-values [ee-embedding-query-prefix "search_query: "]+ (is (= (zipmap model-names (repeat "search_query: hello"))+ (prefix-all model-names))))))))(deftest test-openai-provider-validation(testing "OpenAIProvider throws when API key not configured"@@ -97,6 +130,50 @@(is (zero? (#'embedding/count-tokens "")))(is (nil? (#'embedding/count-tokens nil)))))+(deftest in-process-token-usage-test+ (let [model {:provider "in-process" :model-name "local-model" :vector-dimensions 4}+ unnamed-model (dissoc model :model-name)+ analytics-calls (atom [])+ tracking-calls (atom [])+ resolution-calls (atom [])]+ (with-redefs [embeddings.provider/resolve-model (fn [requested]+ (swap! resolution-calls conj requested)+ (assoc requested :model-name "local-model"))+ embeddings.provider/embed-text (fn [_ text _] [text])+ embeddings.provider/embed-texts (fn [_ texts _] (mapv vector texts))+ analytics/inc! (fn [metric labels value]+ (swap! analytics-calls conj [metric labels value]))+ semantic.models.token-tracking/record-tokens (fn [& args]+ (swap! tracking-calls conj args))]+ (embedding/get-embedding model "Hello world" :type :query :record-tokens? true)+ (embedding/get-embeddings-batch model ["Hello world" "again"] :type :index :record-tokens? true)+ (testing "local calls report approximate token metrics and persistent usage"+ (is (= [[:metabase-search/semantic-embedding-tokens+ {:provider "in-process" :model "local-model"}+ 2]+ [:metabase-search/semantic-embedding-tokens+ {:provider "in-process" :model "local-model"}+ 3]]+ @analytics-calls))+ (is (= [["local-model" :query 2]+ ["local-model" :index 3]]+ @tracking-calls)))+ (testing "named local models rely on the provider's embed-time resolution"+ (is (empty? @resolution-calls)))+ (testing "the adapter neither resolves nor counts other providers"+ (embedding/get-embedding (assoc model :provider "openai") "Hello world")+ (is (empty? @resolution-calls))+ (is (= 2 (count @analytics-calls)))+ (is (= 2 (count @tracking-calls))))+ (testing "an unnamed local request resolves once to obtain its token label"+ (embedding/get-embedding unnamed-model "Hello world" :type :query :record-tokens? true)+ (is (= [unnamed-model] @resolution-calls))+ (is (= [:metabase-search/semantic-embedding-tokens+ {:provider "in-process" :model "local-model"}+ 2]+ (last @analytics-calls)))+ (is (= ["local-model" :query 2] (last @tracking-calls)))))))+(deftest test-batching-logic(testing "create-batches handles empty input"(is (empty? (#'embedding/create-batches 10 count [])))@@ -331,6 +408,32 @@"tag" "embedding_generation"}}]events))))))))+(deftest test-embedding-service-snowplow-suppression+ (testing "ai-service fires no token_usage event when the caller passes :snowplow? false"+ ;; The health and circuit-recovery probes embed a synthetic string; counting those as organic usage+ ;; would inflate the token_usage series by however often the probes run.+ (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://mock-embedding-service"+ ee-embedding-service-api-key "mock-key"]+ (let [mock-response {:data [{:object "embedding"+ :embedding (encode-floats-to-base64 [1.0 2.0 3.0])+ :index 0}]+ :model "test-model"+ :usage {:prompt_tokens 5+ :total_tokens 5}}]+ (snowplow-test/with-fake-snowplow-collector+ (mt/with-dynamic-fn-redefs [http/post (fn [_url & _opts]+ {:status 200+ :headers {"Content-Type" "application/json"}+ :body (json/encode mock-response)})]+ (embedding/get-embedding {:provider "ai-service"+ :model-name "test-model"+ :vector-dimensions 3}+ "health check"+ {:type :query, :record-tokens? false, :snowplow? false}))+ (let [events (->> (snowplow-test/pop-event-data-and-user-id!)+ (filter #(= "embedding_generation" (get-in % [:data "tag"]))))]+ (is (empty? events))))))))+(deftest ^:sequential token-tracking-write-test(mt/with-premium-features #{:semantic-search}(when (string? (not-empty (:mb-pgvector-db-url env/env)))@@ -401,4 +504,17 @@(is (false? (embedding/embedding-supported? {:provider "openai"})))))(testing "ollama is always supported; an unrecognized provider is not (:default)"(is (true? (embedding/embedding-supported? {:provider "ollama"})))- (is (false? (embedding/embedding-supported? {:provider "no-embedder"})))))+ (is (false? (embedding/embedding-supported? {:provider "no-embedder"}))))+ (testing "in-process can be configured before its plugin is installed, but is not ready without it"+ (is (false? (embedding/embedding-supported? {:provider "in-process"})))))++(deftest resolve-model-test+ (let [requested {:provider "openai"+ :model-name "text-embedding-3-small"+ :vector-dimensions 1536}+ resolved (embedding/resolve-model requested)]+ (is (= requested (select-keys resolved (keys requested))))+ (is (= 1 (:embedding-spi-version resolved)))+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}" (:embedding-space-id resolved)))+ (testing "transport credentials are not part of vector-space identity"+ (is (= resolved (embedding/resolve-model (assoc requested :api-key "do-not-persist")))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.cljindex b77feeae078..07ecbf07ee7 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/gate_test.clj@@ -255,8 +255,10 @@(deftest flush-watermark!-test(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)- model1 {:provider "foo" :model-name "m1" :vector-dimensions 42}- model2 {:provider "foo" :model-name "m2" :vector-dimensions 42}+ model1 (semantic.tu/resolved-mock-embedding-model+ :provider "foo" :model-name "m1" :vector-dimensions 42)+ model2 (semantic.tu/resolved-mock-embedding-model+ :provider "foo" :model-name "m2" :vector-dimensions 42)index1 (semantic.index-metadata/qualify-index (semantic.index/default-index model1) index-metadata)index2 (semantic.index-metadata/qualify-index (semantic.index/default-index model2) index-metadata)watermark {:last-poll (ts "2025-01-01T13:00:00Z")diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.cljindex 8b0d0b8bb6a..760b0547cee 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj@@ -26,8 +26,23 @@(def ^:private embedding-model{:provider "ai-service", :model-name "test-model", :vector-dimensions 3})+(def ^:private space-id+ "The embedding space the active index was built in. Health compares this, not the settings keys."+ "emb:v1:sha256:test")++(def ^:private other-space-id+ "A space the same settings could resolve to after a provider upgrade."+ "emb:v1:sha256:old")++(def ^:private active-index-embedding-model+ "The active index's :embedding-model, as produced by index-metadata/row->index: it carries bookkeeping+ keys (:embedding-space-id, :embedding-spi-version) that get-configured-embedding-model never returns."+ (assoc embedding-model+ :embedding-space-id space-id+ :embedding-spi-version 1))+(def ^:private active-state- {:index {:table-name "index_test_table", :embedding-model embedding-model}+ {:index {:table-name "index_test_table", :embedding-model active-index-embedding-model}:metadata-row {:id 1:indexer_stalled_at nil:repair_orphan_count nil@@ -42,6 +57,7 @@semantic.env/get-pgvector-datasource! (constantly ::pgvector)semantic.env/get-index-metadata (constantly ::index-metadata)semantic.env/get-configured-embedding-model (constantly embedding-model)+ semantic.embedding/resolve-model (constantly active-index-embedding-model)semantic.settings/semantic-search-vector-strategy (constantly :brute-force)embedding-health/request-circuit-recovery! (constantly nil)](thunk)))@@ -132,14 +148,22 @@embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})](is (=? {:health 0 :message #".*required HNSW index is missing.*"}(semantic.health/index-health-check)))))- (testing "an active index for a different embedding model degrades without probing the wrong model"+ (testing "an index built in another embedding space degrades without probing the wrong model"(mt/with-dynamic-fn-redefs[semantic.index-metadata/get-active-index-state- (constantly (assoc-in active-state [:index :embedding-model :model-name] "old-model"))+ (constantly (assoc-in active-state [:index :embedding-model :embedding-space-id] other-space-id))semantic.health/active-index-queryable? (constantly true)embedding-health/embedding-problem #(throw (ex-info "must not probe configured model" {}))](is (=? {:health 0 :message #".*active index embedding model does not match configured model.*"}(semantic.health/index-health-check)))))+ (testing "a configured model the provider cannot resolve degrades, and is not read as a mismatch"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/resolve-model (fn [_] (throw (ex-info "no provider" {})))+ embedding-health/embedding-problem #(throw (ex-info "must not probe" {}))]+ (is (=? {:health 0 :message #".*cannot be resolved: no provider.*"}+ (semantic.health/index-health-check)))))(testing "a non-closed breaker (open or half-open) still probes (so a recovered-but-idle embedder isdetectable) and degrades identically in both states"(let [probed? (atom false)]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.cljindex d6ef17f91f8..178c5970a0c 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_embedding_test.clj@@ -10,6 +10,13 @@(def ^:private test-long-model-name "some-really-really-really-really-really-long-model-name-that-will-exceed-the-limit")(def ^:private test-vector-dimensions 1024)+(deftest ^:parallel hash-identifier-uses-postgres-byte-limit-test+ (let [identifier (str "index_" (apply str (repeat 30 "é")))]+ (is (<= (count identifier) 63) "the identifier fits when incorrectly counted as characters")+ (is (re-matches #"index_[0-9a-f]{40}"+ (semantic.index/hash-identifier-if-exceeds-pg-limit identifier))+ "UTF-8 identifiers over PostgreSQL's 63-byte limit are hashed")))+;; Postgres truncates identifier names to 63 bytes (+ 1 byte for a terminating NULL). Ensure that names do not exceed the limit(deftest ^:parallel index-embedding-name-length-test(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.cljindex 74bab719e48..0c8a2f280a5 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_metadata_test.clj@@ -5,6 +5,7 @@[metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.embeddings.provider :as embeddings.provider][metabase.test :as mt][metabase.util :as u])(:import@@ -23,7 +24,19 @@(is (= {:table-name "bar"} (sut {:table-name "bar"} {:index-table-qualifier "%s"})))(testing "qualifier applies to derived index names"(is (= "foo_bar_embed_hnsw_idx" (-> (sut {:table-name "bar"} {:index-table-qualifier "foo_%s"})- semantic.index/hnsw-index-name)))))))+ semantic.index/hnsw-index-name)))))+ (testing "hashes over-long identifiers within Postgres's 63-byte limit"+ (let [long-table (apply str "index_" (repeat 60 "x"))+ fits-table (apply str "index_" (repeat 42 "x"))]+ (testing "an unqualified over-long name is hashed whole"+ (is (= (semantic.index/hash-identifier-if-exceeds-pg-limit long-table)+ (:table-name (sut {:table-name long-table} {:index-table-qualifier "%s"})))))+ (testing "a schema qualifier is preserved; only the table component is hashed"+ (is (= (str "semantic_search." (semantic.index/hash-identifier-if-exceeds-pg-limit long-table))+ (:table-name (sut {:table-name long-table} {:index-table-qualifier "semantic_search.%s"})))))+ (testing "a schema-qualified name over 63 bytes whose table component fits is left intact"+ (is (= (str "semantic_search." fits-table)+ (:table-name (sut {:table-name fits-table} {:index-table-qualifier "semantic_search.%s"})))))))))(deftest create-tables-if-not-exists!-test(let [pgvector (semantic.env/get-pgvector-datasource!)@@ -116,6 +129,22 @@:metadata-row (first (semantic.tu/get-metadata-rows pgvector index-metadata))}(sut pgvector index-metadata))))))))+(deftest revisioned-active-index-state-round-trip-test+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.tu/unique-index-metadata)+ embedding-model (semantic.tu/resolved-mock-embedding-model :model-revision "revision-1")+ index (-> (semantic.index/default-index embedding-model)+ (semantic.index-metadata/qualify-index index-metadata))]+ (with-open [_ (open-tables! pgvector index-metadata)]+ (semantic.index-metadata/ensure-control-row-exists! pgvector index-metadata)+ (let [index-id (semantic.index-metadata/record-new-index-table! pgvector index-metadata index)]+ (semantic.index-metadata/activate-index! pgvector index-metadata index-id)+ (let [{:keys [index metadata-row]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)+ reloaded-model (:embedding-model index)]+ (is (= embedding-model reloaded-model))+ (is (= "revision-1" (:model_revision metadata-row)))+ (is (= 4 (count (embeddings.provider/embed-text reloaded-model "round-trip")))))))))+(defn- default-index [embedding-model index-metadata](mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix semantic.tu/mock-table-suffix](-> (semantic.index/default-index embedding-model)@@ -137,8 +166,7 @@(deftest find-compatible-index!-test(let [pgvector (semantic.env/get-pgvector-datasource!)embedding-model1 semantic.tu/mock-embedding-model- embedding-model2 (assoc semantic.tu/mock-embedding-model- :model-name "mock2")+ embedding-model2 (semantic.tu/resolved-mock-embedding-model :model-name "mock2")sut semantic.index-metadata/find-compatible-index!;; warning: the setup-scenario can currently only set up a happy path;; other variables include:@@ -190,6 +218,16 @@(testing "no metadata"(is (nil? (sut' model))))))))))))+(deftest embedding-space-id-is-part-of-index-compatibility-test+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.tu/unique-index-metadata)+ model semantic.tu/mock-embedding-model]+ (with-open [_ (open-tables! pgvector index-metadata)]+ (setup-scenario! pgvector index-metadata {:active model})+ (is (some? (semantic.index-metadata/find-compatible-index! pgvector index-metadata model)))+ (is (nil? (semantic.index-metadata/find-compatible-index!+ pgvector index-metadata (assoc model :embedding-space-id "replacement-space")))))))+(deftest create-new-index-spec-test(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)@@ -217,7 +255,9 @@(is (=? [{:id index-id:provider (:provider embedding-model):model_name (:model-name embedding-model)+ :model_revision nil:vector_dimensions (:vector-dimensions embedding-model)+ :embedding_space_id (:embedding-space-id embedding-model):table_name (:table-name index):index_version (:version index)}](semantic.tu/get-metadata-rows pgvector index-metadata)))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex dc9431f801a..098d71c1d3b 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -179,6 +179,8 @@;; force-reset suffix, as pgvector-api/fresh-index builds it(str (semantic.index/model-table-name semantic.tu/mock-embedding-model) "_" (semantic.index/model-table-suffix))"index_ollama_mxbai_lg_1024_1234567"+ "index_ollama_mxbai_lg_1024_e0123456789ab"+ "index_ollama_mxbai_lg_1024_e0123456789ab_1234567";; hashed shape for names exceeding the pg identifier limit(semantic.index/hash-identifier-if-exceeds-pg-limit (apply str "index_" (repeat 60 "x")))(str "index_" (apply str (repeat 40 "a")))@@ -203,6 +205,32 @@(str "index_" (apply str (repeat 39 "a")))(str "index_" (apply str (repeat 41 "a"))))))+(deftest ^:parallel model-table-name-embedding-space-test+ (let [model semantic.tu/mock-embedding-model]+ (is (not= (semantic.index/model-table-name model)+ (semantic.index/model-table-name (assoc model :embedding-space-id "another-space"))))+ (is (= (str "index_" (semantic.embedding/abbrev-provider-name "mock") "_model_4")+ (semantic.index/model-table-name (dissoc model :embedding-space-id))))+ (testing "plugin provider identifiers occupy a disjoint, collision-resistant namespace"+ (let [unsafe-name "vendor/foo"+ encoded-name (semantic.embedding/abbrev-provider-name unsafe-name)+ provider-names [unsafe-name "vendor-foo" "vendor foo" encoded-name]+ table-names (mapv #(semantic.index/model-table-name (assoc model :provider %)) provider-names)]+ (is (= (count provider-names) (count (distinct table-names))))+ (is (not= encoded-name (semantic.embedding/abbrev-provider-name encoded-name))+ "an encoded unsafe name must not collide when registered verbatim as another provider")+ (doseq [[built-in alias] [["ai-service" "ais"]+ ["in-process" "inproc"]+ ["ollama" "plugin_ollama"]+ ["openai" "plugin_openai"]]]+ (is (not= (semantic.index/model-table-name (assoc model :provider built-in))+ (semantic.index/model-table-name (assoc model :provider alias)))+ (str "built-in alias must not collide with plugin provider " (pr-str alias))))+ (doseq [table-name table-names]+ (let [table-keyword (keyword table-name)]+ (is (nil? (namespace table-keyword)))+ (is (semantic.index/index-table-name? table-name))))))))+(defn- expected-index-defs"The index-DDL contract of [[semantic.index/create-index-table-if-not-exists!]] for `index`.Maps the name of each index it must create to a pattern its pg_indexes indexdef must match."diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex 147ebf61df6..fa83987398c 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -15,6 +15,7 @@[metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase-enterprise.semantic-search.util :as semantic.util][metabase.collections.models.collection :as collection]+ [metabase.embeddings.provider :as embeddings.provider][metabase.test :as mt][metabase.util :as u][metabase.util.log :as log]@@ -58,6 +59,48 @@(is (m/find-first (comp #{"Migration already performed, skipping."} :message)(:messages <>)))))))))+(deftest version-2-embedding-space-migration-preserves-index-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ table-name "index_legacy_preserved_4"+ model {:provider "openai" :model-name "legacy-model" :vector-dimensions 4}+ space-id (:embedding-space-id (embeddings.provider/legacy-resolved-model model))]+ (jdbc/execute! pgvector ["CREATE TABLE migration (version bigint PRIMARY KEY, migrated_at timestamp DEFAULT NOW(), status varchar(32))"])+ (jdbc/execute! pgvector ["INSERT INTO migration (version, status) VALUES (2, 'success')"])+ (jdbc/execute! pgvector [(str "CREATE TABLE index_metadata ("+ "id bigint PRIMARY KEY, provider text NOT NULL, model_name text NOT NULL, "+ "vector_dimensions int NOT NULL, table_name text NOT NULL UNIQUE, "+ "index_version int NOT NULL, index_created_at timestamptz NOT NULL)")])+ (jdbc/execute! pgvector ["CREATE TABLE index_control (id bigint PRIMARY KEY, version text NOT NULL, active_id int, active_updated_at timestamptz)"])+ (jdbc/execute! pgvector [(format "CREATE TABLE %s (id text PRIMARY KEY)" table-name)])+ (jdbc/execute! pgvector [(str "INSERT INTO index_metadata "+ "(id, provider, model_name, vector_dimensions, table_name, index_version, index_created_at) "+ "VALUES (7, 'openai', 'legacy-model', 4, ?, 5, NOW())")+ table-name])+ (jdbc/execute! pgvector ["INSERT INTO index_control (id, version, active_id) VALUES (0, '2', 7)"])+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate!+ tx {:index-metadata semantic.index-metadata/default-index-metadata}))+ (let [control-row (jdbc/execute-one! pgvector ["SELECT active_id, version FROM index_control"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ metadata-row (jdbc/execute-one! pgvector ["SELECT id, embedding_space_id, model_revision FROM index_metadata"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ column-rows (jdbc/execute! pgvector+ ["SELECT column_name, is_nullable FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'index_metadata' AND column_name IN ('embedding_space_id', 'model_revision')"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ nullable-by-column (into {} (map (juxt :column_name :is_nullable)) column-rows)+ active-model (get-in (semantic.index-metadata/get-active-index-state+ pgvector semantic.index-metadata/default-index-metadata)+ [:index :embedding-model])]+ (testing "the physical index and active pointer survive"+ (is (semantic.util/table-exists? pgvector table-name))+ (is (= {:active_id 7 :version "3"} control-row)))+ (testing "the existing metadata row is backfilled in place"+ (is (= {:id 7 :embedding_space_id space-id :model_revision nil} metadata-row))+ (is (= {"embedding_space_id" "NO" "model_revision" "YES"} nullable-by-column))+ (is (not (contains? active-model :model-revision)))))))))+(deftest schema-scoped-migration-drop-test(mt/with-premium-features #{:semantic-search}(semantic.tu/with-test-db-defaults!@@ -243,7 +286,9 @@:indexer_last_seen:indexer_last_seen_hash:indexer_last_seen_id+ :embedding_space_id:model_name+ :model_revision:provider:table_name:vector_dimensions]))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.cljindex dbc4e623eb5..7e4bcf34292 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj@@ -30,7 +30,7 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)model1 semantic.tu/mock-embedding-model- model2 (assoc semantic.tu/mock-embedding-model :model-name "embed-harder")+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embed-harder")sut* semantic.pgvector-api/init-semantic-search!cleanup (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))sut #(semantic.tu/closeable (apply sut* %&) cleanup)]@@ -179,7 +179,7 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)model1 semantic.tu/mock-embedding-model- model2 (assoc semantic.tu/mock-embedding-model :model-name "embedagain")+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embedagain")sut semantic.pgvector-api/index-documents!](test-not-initialized sut pgvector index-metadata model1)(with-open [index-ref (open-semantic-search! pgvector index-metadata model1)]@@ -207,7 +207,7 @@dash "dashboard"card "card"model1 semantic.tu/mock-embedding-model- model2 (assoc semantic.tu/mock-embedding-model :model-name "embedagain")+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "embedagain")sut semantic.pgvector-api/delete-documents!documents (semantic.tu/mock-documents){card-ids "card", dash-ids "dashboard"} (u/group-by :model :id documents)]@@ -246,7 +246,7 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)model1 semantic.tu/mock-embedding-model- model2 (assoc semantic.tu/mock-embedding-model :model-name "judge-embedd")+ model2 (semantic.tu/resolved-mock-embedding-model :model-name "judge-embedd")remove-scores (fn [rows] (mapv #(dissoc % :score :all-scores) rows)) ; scores have time-sensitives componentssut* semantic.pgvector-api/querysut #(remove-scores (:results (mt/as-admin (apply sut* %&)))) ; see notes below about permsdiff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.cljindex 51ffcd41e43..766f42a5bcd 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_cleanup_test.clj@@ -37,9 +37,12 @@[table-name {:keys [provider model-name vector-dimensions]}]{:table-name table-name:version 1- :embedding-model {:provider provider+ ;; Resolve the model so it carries an :embedding-space-id -- record-new-index-table! persists it into a+ ;; NOT NULL column.+ :embedding-model (semantic.tu/resolved-mock-embedding-model+ :provider provider:model-name model-name- :vector-dimensions vector-dimensions}})+ :vector-dimensions vector-dimensions)})(defn- insert-metadata-with-timestamps!"Insert an index metadata row using record-new-index-table! and then update timestamps for testing."diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.cljindex f5e8629e22c..9639ec66757 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj@@ -17,6 +17,7 @@[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.embeddings.provider :as embeddings.provider][metabase.search.config :as search.config][metabase.search.ingestion :as search.ingestion][metabase.test :as mt]@@ -278,10 +279,18 @@;;;; mock provider+(defn resolved-mock-embedding-model+ "Build a self-consistent resolved descriptor for a mock model variant."+ [& {:as overrides}]+ (-> (merge {:provider "mock"+ :model-name "model"+ :vector-dimensions 4}+ overrides)+ embeddings.provider/legacy-resolved-model+ (assoc :embedding-spi-version embeddings.provider/embedding-spi-version)))+(def mock-embedding-model- {:provider "mock"- :model-name "model"- :vector-dimensions 4})+ (resolved-mock-embedding-model))(def mock-index-metadata"An index metadata to qualify and isolate mock indexes"@@ -306,15 +315,16 @@(def mock-index"A mock index for testing low level indexing functions.Coincides with what the index-metadata system would create for the mock-embedding-model."- (with-redefs [semantic.index/model-table-suffix mock-table-suffix]+ (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix mock-table-suffix](-> (semantic.index/default-index mock-embedding-model)(semantic.index-metadata/qualify-index mock-index-metadata))))-;; NOTE: opts are currently unused in following mock implementations-(defmethod semantic.embedding/get-embedding "mock" [_ text & {:as _opts}] (get-mock-embedding text))-(defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts & {:as _opts}] (get-mock-embeddings-batch texts))-(defmethod semantic.embedding/pull-model "mock" [_])-(defmethod semantic.embedding/embedding-supported? "mock" [_] true)+(embeddings.provider/register-provider!+ "mock"+ {:embedding-spi-version embeddings.provider/embedding-spi-version+ :readiness (constantly {:ready? true})+ :resolve-model embeddings.provider/legacy-resolved-model+ :embed-texts (fn [_ texts _opts] (get-mock-embeddings-batch texts))})#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}(defn query-index [search-context]diff --git a/modules/embedder/README.md b/modules/embedder/README.mdnew file mode 100644index 00000000000..4a3105d5e3a--- /dev/null+++ b/modules/embedder/README.md@@ -0,0 +1,46 @@+# In-process embedding plugin++This module builds `metabase-embedder-plugin.jar`, an optional runtime plugin containing two pinned models:++- `Snowflake/snowflake-arctic-embed-xs` (384 dimensions) — semantic search and Library retrieval+- `sentence-transformers/all-MiniLM-L6-v2` (384 dimensions) — the data complexity score's synonym axis++Both models are deliberately small (~23 MB per architecture) so CI can build the artifact and run real inference+against it on every change without caching multi-GB downloads. They exercise the full packaging, tokenizer, and DJL+runtime machinery. A release build bundling a larger retrieval model (such as `snowflake-arctic-embed-l-v2.0`) will+be added when the plugin is published for customers or offline use such as evals.++Build it with `./bin/build-embedder-plugin.sh`, then place the jar in Metabase's plugin directory. The plugin is+discovered from the jar manifest at startup; each model's DJL runtime is loaded lazily on first inference, and only+the models a consumer actually asks for are ever loaded.++Consumers select the provider through their own settings. Semantic search and Library retrieval must also override+the model and dimensions because their defaults describe the larger Arctic Embed model used by the AI service:++```text+MB_EE_EMBEDDING_PROVIDER=in-process+MB_EE_EMBEDDING_MODEL=Snowflake/snowflake-arctic-embed-xs+MB_EE_EMBEDDING_MODEL_DIMENSIONS=384+```++The data complexity score's default model and dimensions already match the bundled MiniLM model, so it only needs:++```text+MB_DATA_COMPLEXITY_SCORING_SYNONYM_EMBEDDING_PROVIDER=in-process+```++A model the plugin does not bundle, or configured dimensions that do not match it, fails readiness rather than+falling back to another model.++The artifact contains pinned ARM64 and x86-64/AVX2 ONNX exports. The supported runtime combinations are glibc 2.34+or newer on Linux, on either architecture, and Apple Silicon macOS. Intel macOS is unsupported because the tokenizer+dependency does not ship an x86-64 macOS native library. The runtime is not musl-compatible, so it cannot run in+Metabase's default Alpine image; use a glibc-based image such as the repository's Ubuntu image when installing this+plugin. Other operating systems, libc implementations, and architectures fail readiness checks before model loading.++The exact embedding-space identity includes the architecture-specific export digest, so every node sharing+semantic-search or library-retrieval indexes must use the same architecture. Mixed ARM64/x86-64 clusters fail closed+rather than querying vectors produced by a different export.++Custom model sources are intentionally not supported in this first version: the bundled catalog is the only place a+model can come from.diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex 2e6abc43268..ca292b8fe89 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -16,6 +16,7 @@[metabase.driver.mysql][metabase.driver.postgres][metabase.embedding.settings :as embed.settings]+ [metabase.embeddings.startup :as embeddings.startup][metabase.events.core :as events][metabase.initialization-status.core :as init-status][metabase.llm.startup :as llm.startup]@@ -158,12 +159,6 @@(catch Exception e(log/warnf "Failed to register signal handler for SIG%s: %s" signal-name (ex-message e)))))))-(def embedder-plugin-name- "Manifest `info.name` of the in-process embedder. Pinned against the manifest by- `metabase.embeddings.embedder-plugin-name-test`; a rename here without one there silently disables- the plugin, since `plugins/registered?` looks it up by this exact string."- "Metabase In-Process Embedder")-(defn- init!*"General application initialization function which should be run once at application startup."[]@@ -184,13 +179,7 @@(tracing/init!);; load any plugins as needed(plugins/load-plugins!)- ;; The in-process embedder registers its provider during plugin initialization. Its model runtime remains- ;; lazy, so activating the lightweight registration namespace here does not load DJL or a model at startup.- (when (plugins/registered? embedder-plugin-name)- (try- (plugins/load-plugin! embedder-plugin-name)- (catch Exception e- (log/warnf "Unable to activate the in-process embedder plugin: %s" (ex-message e)))))+ (embeddings.startup/ensure-in-process-provider!)(init-status/set-progress! 0.3)(setting/validate-settings-formatting!);; startup database. validates connection & runs any necessary migrationsdiff --git a/src/metabase/embeddings/startup.clj b/src/metabase/embeddings/startup.cljnew file mode 100644index 00000000000..3440768f3aa--- /dev/null+++ b/src/metabase/embeddings/startup.clj@@ -0,0 +1,31 @@+(ns metabase.embeddings.startup+ "Startup-time registration of the in-process embedding provider.++ The provider registers itself while its plugin initializes, so every entry point that might embed has to run+ this. The server does it during init; standalone modes that skip init have to do it themselves, or embedding+ fails with `:provider-not-registered`."+ (:require+ [metabase.plugins.core :as plugins]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(def embedder-plugin-name+ "Manifest `info.name` of the in-process embedder.+ Pinned against the manifest by `metabase.embeddings.embedder-plugin-name-test`; a rename here without+ one there silently disables the plugin, since `plugins/registered?` looks it up by this exact string."+ "Metabase In-Process Embedder")++(defn ensure-in-process-provider!+ "Register the in-process embedding provider when its plugin is installed.++ Plugin loading is idempotent, so callers that have already loaded plugins pay nothing extra. Initialization only+ registers the provider: DJL and the model itself stay lazy, so a server that never embeds loads neither. The+ plugin ships separately from the uberjar, so its absence is the normal case and is not worth a warning."+ []+ (plugins/load-plugins!)+ (when (plugins/registered? embedder-plugin-name)+ (try+ (plugins/load-plugin! embedder-plugin-name)+ (catch Exception e+ (log/warnf "Unable to activate the in-process embedder plugin: %s" (ex-message e))))))diff --git a/test/metabase/embeddings/embedder_plugin_artifact_test.clj b/test/metabase/embeddings/embedder_plugin_artifact_test.cljindex 3ccbf87a56b..ca9b937ca2a 100644--- a/test/metabase/embeddings/embedder_plugin_artifact_test.clj+++ b/test/metabase/embeddings/embedder_plugin_artifact_test.clj@@ -3,8 +3,8 @@[clojure.java.io :as io][clojure.string :as str][clojure.test :refer :all]- [metabase.core.core :as core][metabase.embeddings.provider :as embeddings.provider]+ [metabase.embeddings.startup :as embeddings.startup][metabase.plugins.impl :as plugins][metabase.plugins.initialize :as plugin-initialize])(:import@@ -12,11 +12,14 @@(set! *warn-on-reflection* true)-(def ^:private requested-model+(def ^:private retrieval-model+ "What semantic search and Library retrieval ask for."{:provider "in-process":model-name "Snowflake/snowflake-arctic-embed-xs"})-(def ^:private requested-minilm-model+(def ^:private complexity-model+ "What the data-complexity score's synonym axis asks for. Mean-pooled — unlike the cls-pooled retrieval model —+ so it exercises a different translator configuration."{:provider "in-process":model-name "sentence-transformers/all-MiniLM-L6-v2"})@@ -65,6 +68,61 @@(java.util.Collections/singletonList Proxy/NO_PROXY))(connectFailed [_uri _socket-address _exception])))+(defn- check-platform-support!+ "Which platforms the bundled natives claim to support, driven through the catalog's private probes."+ []+ (let [libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)+ os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)+ arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)+ detect-libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'detect-linux-libc)+ detect-libc (var-get detect-libc-var)+ glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)+ supported-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'supported-glibc-version?)+ supported-version? (var-get supported-version-var)]+ (is (= :unknown (detect-libc false nil)) "unreadable process maps fail closed")+ (is (= :musl (detect-libc true nil)))+ (is (= :musl (detect-libc false "/lib/ld-musl-x86_64.so.1")))+ (is (= :glibc (detect-libc false "/usr/lib64/ld-2.28.so")))+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2"))+ "the Ubuntu 24.04 loader layout is recognized")+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/libc.so.6")))+ (is (false? (supported-version? "2.33")))+ (is (true? (supported-version? "2.34")))+ (is (true? (supported-version? "2.39")))+ (is (false? (supported-version? nil)))+ (with-redefs-fn {libc-var (constantly :glibc)+ os-var (constantly "linux")+ glibc-version-var (constantly "2.34")}+ #(is (true? (:ready? (embeddings.provider/readiness retrieval-model)))))+ (with-redefs-fn {libc-var (constantly :unknown)+ os-var (constantly "linux")+ glibc-version-var (constantly "2.39")}+ #(is (true? (:ready? (embeddings.provider/readiness retrieval-model)))+ "the authoritative glibc API probe survives an inconclusive process map"))+ (with-redefs-fn {libc-var (constantly :glibc)+ os-var (constantly "linux")+ glibc-version-var (constantly "2.33")}+ #(do+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc-version}+ (embeddings.provider/readiness retrieval-model)))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc-version"+ (embeddings.provider/resolve-model retrieval-model)))))+ (doseq [[unsupported-libc probed-glibc-version] [[:musl "2.39"]+ [:unknown nil]]]+ (with-redefs-fn {libc-var (constantly unsupported-libc)+ os-var (constantly "linux")+ glibc-version-var (constantly probed-glibc-version)}+ #(do+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc}+ (embeddings.provider/readiness retrieval-model)))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc"+ (embeddings.provider/resolve-model retrieval-model))))))+ (with-redefs-fn {os-var (constantly "mac os x")+ arch-var (constantly "avx2")}+ #(is (= {:provider "in-process" :ready? false :reason :unsupported-platform}+ (embeddings.provider/readiness retrieval-model))+ "the tokenizer artifact has no Intel macOS native library"))))+(defn- check-missing-architecture!"A catalog entry with no export for this architecture must fail rather than hash a nil sha256 into awell-formed but meaningless embedding-space id."@@ -86,12 +144,12 @@(is true))(let [pre-plugin-thread(isolated-thread-call- #(hash-map :readiness (embeddings.provider/readiness requested-model)- :resolved (embeddings.provider/resolve-model requested-model)))]+ #(hash-map :readiness (embeddings.provider/readiness retrieval-model)+ :resolved (embeddings.provider/resolve-model retrieval-model)))](plugins/load-plugins!);; Non-driver plugins are registered eagerly but initialized only when a feature selects them. The provider;; registration lives in the plugin init namespace, so activate this artifact explicitly before exercising it.- (plugin-initialize/load-plugin! core/embedder-plugin-name)+ (plugin-initialize/load-plugin! embeddings.startup/embedder-plugin-name)(testing "the implementation was activated through its jar manifest"(is (embeddings.provider/registered? "in-process"))(is (= "true" (System/getProperty "ai.djl.offline")))@@ -105,15 +163,15 @@(is (nil? (find-ns 'metabase-enterprise.embedder.model))"catalog access from an old worker thread remains lazy")))(testing "the built artifact contains a ready, immutable model space"- (is (true? (:ready? (embeddings.provider/readiness requested-model))))+ (is (true? (:ready? (embeddings.provider/readiness retrieval-model))))(is (= {:provider "in-process" :ready? false :reason :vector-dimensions-mismatch}(embeddings.provider/readiness- (assoc requested-model :vector-dimensions 768))))+ (assoc retrieval-model :vector-dimensions 768))))(is (re-matches #"emb:v1:sha256:[0-9a-f]{64}"- (:embedding-space-id (embeddings.provider/resolve-model requested-model))))+ (:embedding-space-id (embeddings.provider/resolve-model retrieval-model))))(is (thrown-with-msg? clojure.lang.ExceptionInfo #"has 384 dimensions"(embeddings.provider/resolve-model- (assoc requested-model :vector-dimensions 768))))+ (assoc retrieval-model :vector-dimensions 768))))(let [architecture-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)@@ -123,7 +181,7 @@os-var (constantly "linux")libc-var (constantly :glibc)glibc-version-var (constantly "2.34")}- #(embeddings.provider/resolve-model requested-model)))+ #(embeddings.provider/resolve-model retrieval-model)))normal (mapv (comp :embedding-space-id resolve-for) ["arm64" "avx2"])bounded (binding [*print-length* 1*print-level* 1@@ -135,66 +193,28 @@"architecture-specific exports intentionally have distinct embedding spaces"))(let [model-spec-var (ns-resolve 'metabase-enterprise.embedder.catalog 'model-spec)model-spec-fn (var-get model-spec-var)- spec (model-spec-fn (:model-name requested-model))- normal (embeddings.provider/resolve-model requested-model)+ spec (model-spec-fn (:model-name retrieval-model))+ normal (embeddings.provider/resolve-model retrieval-model)reordered (update spec :runtime #(into (array-map) (reverse %)))equivalent (with-redefs-fn {model-spec-var (constantly reordered)}- #(embeddings.provider/resolve-model requested-model))]+ #(embeddings.provider/resolve-model retrieval-model))](is (= (:embedding-space-id normal) (:embedding-space-id equivalent))"equivalent catalog map order does not change model identity")- (check-missing-architecture! requested-model))- (let [libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)- os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)- detect-libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'detect-linux-libc)- detect-libc (var-get detect-libc-var)- glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)- supported-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'supported-glibc-version?)- supported-version? (var-get supported-version-var)]- (is (= :unknown (detect-libc false nil)) "unreadable process maps fail closed")- (is (= :musl (detect-libc true nil)))- (is (= :musl (detect-libc false "/lib/ld-musl-x86_64.so.1")))- (is (= :glibc (detect-libc false "/usr/lib64/ld-2.28.so")))- (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2"))- "the Ubuntu 24.04 loader layout is recognized")- (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/libc.so.6")))- (is (false? (supported-version? "2.33")))- (is (true? (supported-version? "2.34")))- (is (true? (supported-version? "2.39")))- (is (false? (supported-version? nil)))- (with-redefs-fn {libc-var (constantly :glibc)- os-var (constantly "linux")- glibc-version-var (constantly "2.34")}- #(is (true? (:ready? (embeddings.provider/readiness requested-model)))))- (with-redefs-fn {libc-var (constantly :unknown)- os-var (constantly "linux")- glibc-version-var (constantly "2.39")}- #(is (true? (:ready? (embeddings.provider/readiness requested-model)))- "the authoritative glibc API probe survives an inconclusive process map"))- (with-redefs-fn {libc-var (constantly :glibc)- os-var (constantly "linux")- glibc-version-var (constantly "2.33")}- #(do- (is (= {:provider "in-process" :ready? false :reason :unsupported-libc-version}- (embeddings.provider/readiness requested-model)))- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc-version"- (embeddings.provider/resolve-model requested-model)))))- (doseq [[unsupported-libc probed-glibc-version] [[:musl "2.39"]- [:unknown nil]]]- (with-redefs-fn {libc-var (constantly unsupported-libc)- os-var (constantly "linux")- glibc-version-var (constantly probed-glibc-version)}- #(do- (is (= {:provider "in-process" :ready? false :reason :unsupported-libc}- (embeddings.provider/readiness requested-model)))- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc"- (embeddings.provider/resolve-model requested-model)))))))- (let [os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)- arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)]- (with-redefs-fn {os-var (constantly "mac os x")- arch-var (constantly "avx2")}- #(is (= {:provider "in-process" :ready? false :reason :unsupported-platform}- (embeddings.provider/readiness requested-model))- "the tokenizer artifact has no Intel macOS native library"))))+ (check-missing-architecture! retrieval-model))+ (check-platform-support!))+ (testing "the same artifact serves the complexity score's model in its own vector space"+ (is (true? (:ready? (embeddings.provider/readiness complexity-model))))+ (is (= {:provider "in-process" :ready? false :reason :vector-dimensions-mismatch}+ (embeddings.provider/readiness (assoc complexity-model :vector-dimensions 1024))))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"has 384 dimensions"+ (embeddings.provider/resolve-model+ (assoc complexity-model :vector-dimensions 1024))))+ (let [resolved (embeddings.provider/resolve-model complexity-model)]+ (is (= 384 (:vector-dimensions resolved)))+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}" (:embedding-space-id resolved)))+ (is (not= (:embedding-space-id (embeddings.provider/resolve-model retrieval-model))+ (:embedding-space-id resolved))+ "the two bundled models must never share an embedding space")))(testing "the artifact bounds local inference batches before loading the runtime"(let [model-fn-var (ns-resolve 'metabase-enterprise.embedder.plugin 'model-fn)batch-sizes (atom [])]@@ -203,7 +223,7 @@(fn [_model-name texts](swap! batch-sizes conj (count texts))(mapv (fn [_] (float-array 384)) texts)))}- #(embeddings.provider/embed-texts requested-model (repeat 65 "text")))+ #(embeddings.provider/embed-texts retrieval-model (repeat 65 "text")))(is (= [32 32 1] @batch-sizes))(is (nil? (find-ns 'metabase-enterprise.embedder.model)))))(testing "real inference runs through the plugin jar"@@ -214,17 +234,23 @@(let [[dog puppy invoice](await-thread!(isolated-thread-call- #(embeddings.provider/embed-texts requested-model ["dog" "puppy" "invoice"])))- minilm-embeddings- (await-thread!- (isolated-thread-call- #(embeddings.provider/embed-texts requested-minilm-model ["dog" "invoice"])))]+ #(embeddings.provider/embed-texts retrieval-model ["dog" "puppy" "invoice"])))](is (some? (find-ns 'metabase-enterprise.embedder.model)))(is (= [384 384 384] (mapv alength [dog puppy invoice])))- (is (> (cosine dog puppy) (cosine dog invoice)))- (is (= [384 384] (mapv alength minilm-embeddings)))- (is (every? #(< (Math/abs (- 1.0 (magnitude %))) 1.0e-5) minilm-embeddings)- "MiniLM produces normalized 384-dimensional embeddings"))+ (is (> (cosine dog puppy) (cosine dog invoice))))+ (testing "the complexity-score model infers from the same loaded artifact"+ (let [[monthly-active-users daily-active-users invoice]+ (await-thread!+ (isolated-thread-call+ #(embeddings.provider/embed-texts+ complexity-model+ ["monthly active users" "daily active users" "invoice"])))]+ (is (= [384 384 384] (mapv alength [monthly-active-users daily-active-users invoice])))+ (is (every? #(< (Math/abs (- 1.0 (magnitude %))) 1.0e-5)+ [monthly-active-users daily-active-users invoice])+ "MiniLM produces normalized embeddings")+ (is (> (cosine monthly-active-users daily-active-users)+ (cosine monthly-active-users invoice)))))(finally(ProxySelector/setDefault original-proxy)))(is (empty? @network-attempts) "bundled inference performs no outbound requests"))))))diff --git a/test/metabase/embeddings/embedder_plugin_name_test.clj b/test/metabase/embeddings/embedder_plugin_name_test.cljindex 6962c377463..cad310d8ff7 100644--- a/test/metabase/embeddings/embedder_plugin_name_test.clj+++ b/test/metabase/embeddings/embedder_plugin_name_test.clj@@ -1,14 +1,14 @@(ns metabase.embeddings.embedder-plugin-name-test(:require[clojure.test :refer :all]- [metabase.core.core :as core]+ [metabase.embeddings.startup :as embeddings.startup][metabase.util.yaml :as yaml]))(set! *warn-on-reflection* true)(deftest embedder-plugin-name-matches-manifest-test- (testing "core/embedder-plugin-name is what plugins/registered? looks up by, so it must match the manifest"+ (testing "embedder-plugin-name matches the manifest's info.name"(is (= (get-in (yaml/parse-string(slurp "modules/embedder/resources/metabase/embedder/metabase-plugin.yaml"))[:info :name])- core/embedder-plugin-name))))+ embeddings.startup/embedder-plugin-name))))