Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add embedding provider SPI and Arctic embedder plugin (#78310)

Chris Truter2550ef763e57
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 67438efba20..920cbde8ce8 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -990,6 +990,11 @@
util}
:model-imports #{:model/Card :model/Dashboard :model/DashboardCard :model/EmbeddingTheme}}
+ embeddings
+ {:team "Metabot"
+ :api #{metabase.embeddings.provider}
+ :uses #{util}}
+
entity-retrieval
{:team "Metabot"
:api #{metabase.entity-retrieval.core
@@ -3314,6 +3319,11 @@
settings
util}}
+ enterprise/embedder
+ {:team "Metabot"
+ :api #{}
+ :uses #{classloader embeddings util}}
+
enterprise/embedding-hub
{:team "Embedding"
:api #{metabase-enterprise.embedding-hub.api}
diff --git a/.clj-kondo/src/hooks/common/modules.clj b/.clj-kondo/src/hooks/common/modules.clj
index 0a5f439ba49..3f6dd7af6d4 100644
--- a/.clj-kondo/src/hooks/common/modules.clj
+++ b/.clj-kondo/src/hooks/common/modules.clj
@@ -71,7 +71,7 @@
(let [module (module ns-symb)
module-api-namespaces (module-api-namespaces config module)
module-friends (module-friends config module)]
- (or (empty? module-api-namespaces)
+ (or (nil? module-api-namespaces)
(contains? module-api-namespaces ns-symb)
(contains? module-friends current-module))))
diff --git a/.clj-kondo/test/hooks/clojure/core/ns_test.clj b/.clj-kondo/test/hooks/clojure/core/ns_test.clj
index f196ed75517..1a370553b8b 100644
--- a/.clj-kondo/test/hooks/clojure/core/ns_test.clj
+++ b/.clj-kondo/test/hooks/clojure/core/ns_test.clj
@@ -43,6 +43,16 @@
'{:metabase/modules {api {:uses #{search}}
search {:api #{metabase.search.core}}}}))))
+(deftest ^:parallel module-checker-empty-api-test
+ (is (=? [{:message "Namespace metabase.embeddings.core is not an allowed external API namespace for the embeddings module. [:metabase/modules embeddings :api]"
+ :type :metabase/modules}]
+ (lint-modules
+ '(ns metabase.metabot.core
+ (:require
+ [metabase.embeddings.core :as embeddings]))
+ '{:metabase/modules {metabot {:uses #{embeddings}}
+ embeddings {:api #{}}}}))))
+
(deftest ^:parallel module-checker-friends-test
(is (= []
(lint-modules
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 9eb83e726d3..18f9330ffd0 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -222,6 +222,20 @@ cljs:
- ".github/workflows/cljs.yml"
- "shadow-cljs.edn"
+embedder_plugin:
+ - ".github/actions/prepare-backend/**"
+ - ".github/file-paths.yaml"
+ - ".github/workflows/build-scripts.yml"
+ - "bin/build-embedder-plugin.sh"
+ - "bin/build/src/build/embedder_model.clj"
+ - "bin/build/src/build/plugin_uberjar.clj"
+ - "bin/build/src/build_embedder_plugin.clj"
+ - "deps.edn"
+ - "modules/embedder/**"
+ - "src/metabase/embeddings/provider.clj"
+ - "src/metabase/plugins/**"
+ - "test/metabase/embeddings/embedder_plugin_artifact_test.clj"
+
serialization_baseline:
- ".github/workflows/serialization.yml"
- "test_resources/serialization_baseline/**"
diff --git a/.github/workflows/build-scripts.yml b/.github/workflows/build-scripts.yml
index 4cab4b8d235..04497d56430 100644
--- a/.github/workflows/build-scripts.yml
+++ b/.github/workflows/build-scripts.yml
@@ -9,6 +9,50 @@ on:
jobs:
+ files-changed:
+ name: Check which files changed
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 3
+ permissions:
+ contents: read
+ outputs:
+ embedder_plugin: ${{ steps.changes.outputs.embedder_plugin }}
+ steps:
+ - uses: actions/checkout@v6
+ - name: Test which files changed
+ uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+ id: changes
+ with:
+ token: ${{ github.token }}
+ filters: .github/file-paths.yaml
+
+ test-embedder-plugin:
+ needs: files-changed
+ if: needs.files-changed.outputs.embedder_plugin == 'true'
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@v6
+ - name: Prepare back-end environment
+ uses: ./.github/actions/prepare-backend
+ - name: Build embedder plugin
+ run: ./bin/build-embedder-plugin.sh
+ - name: Verify packaged plugin resources
+ run: |
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase/embedder/metabase-plugin.yaml
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase/embedder-model-catalog.edn
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-arm64.zip
+ jar tf modules/embedder/target/metabase-embedder-plugin.jar | grep -Fx metabase-embedder/snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f-avx2.zip
+ - name: Run artifact-only plugin smoke test
+ env:
+ MB_EMBEDDER_ARTIFACT_TEST: "true"
+ MB_PLUGINS_DIR: modules/embedder/target
+ run: >-
+ ./bin/test-agent --oss :only
+ '[metabase.embeddings.embedder-plugin-artifact-test]'
+
test-build-scripts:
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
steps:
diff --git a/.gitignore b/.gitignore
index 62931eeacbf..fb481387334 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,6 +77,9 @@ cypress.env.json
/logs
/modules/drivers/*/resources/namespaces.edn
/modules/drivers/*/target
+# Generated in-process embedder model bundles and build outputs.
+/modules/embedder/resources/metabase-embedder
+/modules/embedder/target
/plugins
/process.yml
/reset-password-artifacts
diff --git a/bin/build-embedder-plugin.sh b/bin/build-embedder-plugin.sh
new file mode 100755
index 00000000000..8fef9298853
--- /dev/null
+++ b/bin/build-embedder-plugin.sh
@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+set -eo pipefail
+
+script_directory=`dirname "${BASH_SOURCE[0]}"`
+cd "$script_directory/.."
+
+source "./bin/check-clojure-cli.sh"
+check_clojure_cli
+
+source "./bin/clear-outdated-cpcaches.sh"
+clear_outdated_cpcaches
+
+clojure -X:build:build/embedder-plugin "$@"
diff --git a/bin/build/src/build/embedder_model.clj b/bin/build/src/build/embedder_model.clj
new file mode 100644
index 00000000000..5ddcf03b11e
--- /dev/null
+++ b/bin/build/src/build/embedder_model.clj
@@ -0,0 +1,105 @@
+(ns build.embedder-model
+ "Fetch, verify, and package the pinned Arctic model bundles used by the embedder plugin."
+ (:require
+ [clojure.edn :as edn]
+ [clojure.java.io :as io]
+ [environ.core :as env]
+ [metabuild-common.core :as u])
+ (:import
+ (java.io File)
+ (java.net URL)
+ (java.nio.file FileVisitOption Files LinkOption Path)
+ (java.security DigestInputStream MessageDigest)
+ (java.util.zip ZipEntry ZipOutputStream)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private catalog-file
+ (io/file u/project-root-directory "modules/embedder/resources/metabase/embedder-model-catalog.edn"))
+
+(defn model-catalog
+ "Read the immutable runtime/build catalog shared with the plugin."
+ []
+ (edn/read-string (slurp catalog-file)))
+
+(def ^:private download-dir
+ (io/file u/project-root-directory "modules/embedder/target/model-download"))
+
+(def ^:private bundle-dir
+ (io/file u/project-root-directory "modules/embedder/resources/metabase-embedder"))
+
+(defn- sha256
+ ^String [^File file]
+ (let [digest (MessageDigest/getInstance "SHA-256")]
+ (with-open [in (DigestInputStream. (io/input-stream file) digest)]
+ (io/copy in (java.io.OutputStream/nullOutputStream)))
+ (format "%064x" (BigInteger. 1 (.digest digest)))))
+
+(defn- verify-sha256!
+ [^File file expected]
+ (let [actual (sha256 file)]
+ (when-not (= expected actual)
+ (throw (ex-info (format "sha256 mismatch for %s: expected %s, got %s" file expected actual)
+ {:file (str file) :expected expected :actual actual})))))
+
+(defn- cached-file
+ ^File [model-name repo-path]
+ (io/file download-dir model-name (u/filename repo-path)))
+
+(defn- download!
+ [hf-repo revision repo-path ^File destination]
+ (let [url (format "https://huggingface.co/%s/resolve/%s/%s" hf-repo revision repo-path)]
+ (u/announce "Downloading %s" url)
+ (io/make-parents destination)
+ (let [connection (doto (.openConnection ^URL (io/as-url url))
+ (.setConnectTimeout 10000)
+ (.setReadTimeout 30000))]
+ (with-open [in (.getInputStream connection)]
+ (io/copy in destination)))))
+
+(defn- fetch-file!
+ ^File [model-name hf-repo revision repo-path expected-sha]
+ (let [file (cached-file model-name repo-path)]
+ (if (and (.exists file) (= expected-sha (sha256 file)))
+ (u/announce "Using cached %s" (str file))
+ (do
+ (download! hf-repo revision repo-path file)
+ (verify-sha256! file expected-sha)))
+ file))
+
+(defn- write-zip!
+ [model-name {:keys [bundle-name model-revision architectures tokenizer-files]} arch ^File zip-file]
+ (let [hf-repo model-name
+ weights (get architectures arch)
+ bundle-files (cons ["model.onnx"
+ (fetch-file! bundle-name hf-repo model-revision
+ (:source weights) (:sha256 weights))]
+ (for [[repo-path expected-sha] tokenizer-files]
+ [repo-path
+ (fetch-file! bundle-name hf-repo model-revision repo-path expected-sha)]))]
+ (io/make-parents zip-file)
+ (with-open [out (ZipOutputStream. (io/output-stream zip-file))]
+ (doseq [[entry-name ^File file] bundle-files]
+ (.putNextEntry out (ZipEntry. ^String entry-name))
+ (io/copy file out)
+ (.closeEntry out)))
+ (u/announce "Wrote %s" (str zip-file))))
+
+(defn- clear-bundle-dir!
+ []
+ (let [root (.toPath ^File bundle-dir)]
+ (when (Files/exists root (into-array LinkOption [LinkOption/NOFOLLOW_LINKS]))
+ (with-open [paths (Files/walk root (make-array FileVisitOption 0))]
+ (doseq [^Path path (sort-by #(.getNameCount ^Path %) > (iterator-seq (.iterator paths)))]
+ (Files/delete path))))))
+
+(defn fetch-model!
+ "Build both architecture bundles. `SKIP_EMBEDDER_MODEL=true` produces no bundles."
+ [_]
+ (u/step "Fetch embedder model and assemble bundles"
+ (clear-bundle-dir!)
+ (when-not (= (env/env :skip-embedder-model) "true")
+ (doseq [[model-name model-spec] (:models (model-catalog))
+ arch (keys (:architectures model-spec))]
+ (write-zip! model-name model-spec arch
+ (io/file bundle-dir (str (:bundle-name model-spec) "-" arch ".zip")))))))
diff --git a/bin/build/src/build_embedder_plugin.clj b/bin/build/src/build_embedder_plugin.clj
new file mode 100644
index 00000000000..71bf314eb8c
--- /dev/null
+++ b/bin/build/src/build_embedder_plugin.clj
@@ -0,0 +1,41 @@
+(ns build-embedder-plugin
+ "Build the manifest-based in-process embedder plugin jar."
+ (:require
+ [build.embedder-model :as embedder-model]
+ [build.plugin-uberjar :as plugin-uberjar]
+ [clojure.tools.build.api :as b]
+ [clojure.tools.build.tasks.uber]
+ [metabuild-common.core :as u]
+ [org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private project-dir
+ (u/filename u/project-root-directory "modules" "embedder"))
+
+(def jar-path
+ "Output path used by packaging and artifact-only smoke tests."
+ (u/filename project-dir "target" "metabase-embedder-plugin.jar"))
+
+(defn- plugin-basis
+ []
+ (-> (plugin-uberjar/project-basis project-dir)
+ (plugin-uberjar/prune-provided-libs! @plugin-uberjar/metabase-core-provided-libs)
+ (dissoc :deps :aliases :mvn/repos)))
+
+(defn build-plugin!
+ "Fetch pinned bundles and assemble the separately installable plugin jar."
+ [_]
+ (u/step "Build metabase-embedder-plugin.jar"
+ (embedder-model/fetch-model! nil)
+ (let [class-dir (u/filename project-dir "target" "classes")]
+ (b/delete {:path class-dir})
+ (b/copy-dir {:src-dirs [(u/filename project-dir "src")
+ (u/filename project-dir "resources")]
+ :target-dir class-dir})
+ (b/uber {:class-dir class-dir
+ :uber-file jar-path
+ :basis (plugin-basis)
+ :conflict-handlers log4j2-conflict-handler
+ :exclude ["META-INF/LICENSE"]})
+ (u/announce "Created %s" jar-path))))
diff --git a/bin/build/test/build/embedder_model_test.clj b/bin/build/test/build/embedder_model_test.clj
new file mode 100644
index 00000000000..dbb40e47378
--- /dev/null
+++ b/bin/build/test/build/embedder_model_test.clj
@@ -0,0 +1,38 @@
+(ns build.embedder-model-test
+ (:require
+ [build.embedder-model :as embedder-model]
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]))
+
+(deftest model-catalog-test
+ (let [{:keys [catalog-version models]} (embedder-model/model-catalog)]
+ (is (= 1 catalog-version))
+ (is (= #{"Snowflake/snowflake-arctic-embed-xs"
+ "sentence-transformers/all-MiniLM-L6-v2"}
+ (set (keys models))))
+ (is (= {"Snowflake/snowflake-arctic-embed-xs"
+ {:vector-dimensions 384
+ :runtime {:inference-contract-version 1
+ :djl-version "0.36.0"
+ :engine "OnnxRuntime"
+ :pooling "cls"
+ :normalize? true
+ :include-token-types? true}}
+ "sentence-transformers/all-MiniLM-L6-v2"
+ {:vector-dimensions 384
+ :runtime {:inference-contract-version 1
+ :djl-version "0.36.0"
+ :engine "OnnxRuntime"
+ :pooling "mean"
+ :normalize? true
+ :include-token-types? true}}}
+ (update-vals models #(select-keys % [:vector-dimensions :runtime]))))
+ (doseq [[model-name model] models]
+ (is (= #{"arm64" "avx2"} (set (keys (:architectures model)))))
+ (is (re-matches #"[0-9a-f]{40}" (:model-revision model)))
+ (is (str/ends-with? (:bundle-name model) (:model-revision model))
+ "the DJL resource URL must change when the pinned model revision changes")
+ (testing (str model-name " has a pinned SHA-256 for every downloaded artifact")
+ (is (every? #(re-matches #"[0-9a-f]{64}" %)
+ (concat (vals (:tokenizer-files model))
+ (map :sha256 (vals (:architectures model))))))))))
diff --git a/deps.edn b/deps.edn
index 89bf6a0b49a..55e6b5b0b33 100644
--- a/deps.edn
+++ b/deps.edn
@@ -866,6 +866,12 @@
:build/verify-driver
{:exec-fn verify-driver/verify-driver}
+ ;; Build the separately installed in-process embedder plugin jar.
+ ;;
+ ;; clojure -X:build:build/embedder-plugin
+ :build/embedder-plugin
+ {:exec-fn build-embedder-plugin/build-plugin!}
+
;; Build i18n artifacts.
;;
;; clojure -X:build:build/i18n
diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.clj
index 7f82551519e..6bfbb957334 100644
--- a/dev/src/dev/deps_graph.clj
+++ b/dev/src/dev/deps_graph.clj
@@ -69,15 +69,20 @@
^java.io.File []
(io/file (str (.getAbsolutePath (project-root-directory)) "/enterprise/backend/src")))
-(mu/defn- drivers-source-roots :- [:sequential (ms/InstanceOfClass java.io.File)]
+(mu/defn- plugin-source-roots :- [:sequential (ms/InstanceOfClass java.io.File)]
+ "Source roots of the separately built plugin modules under `modules/`: every driver, plus the siblings that
+ ship as their own jar. They are off the main classpath, so nothing else here would find them."
[]
- (for [file (.listFiles (io/file (str (.getAbsolutePath (project-root-directory)) "/modules/drivers")))]
- (io/file file "src")))
+ (let [modules (io/file (str (.getAbsolutePath (project-root-directory)) "/modules"))]
+ (concat
+ (for [driver (.listFiles (io/file modules "drivers"))]
+ (io/file driver "src"))
+ [(io/file modules "embedder" "src")])))
(mu/defn- find-source-files :- [:sequential (ms/InstanceOfClass java.io.File)]
[]
(mapcat ns.find/find-sources-in-dir
- (list* (source-root) (enterprise-source-root) (drivers-source-roots))))
+ (list* (source-root) (enterprise-source-root) (plugin-source-roots))))
(mu/defn- module :- [:maybe symbol?]
"E.g.
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 6d699ac8232..112f22fef3c 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -273,9 +273,7 @@
[identifier]
(if (<= (count identifier) 63)
identifier
- (let [hashed-name (str "index_" (buddy-codecs/bytes->hex (buddy-hash/sha1 identifier)))]
- (log/warnf "Using hashed name for index table %s as original table name %s exceeded the maximum table name length" hashed-name identifier)
- hashed-name)))
+ (str "index_" (buddy-codecs/bytes->hex (buddy-hash/sha1 identifier)))))
(defn model-table-suffix
"Returns a new suffix for a table name, based on current timestamp"
diff --git a/modules/embedder/deps.edn b/modules/embedder/deps.edn
new file mode 100644
index 00000000000..3855abb5212
--- /dev/null
+++ b/modules/embedder/deps.edn
@@ -0,0 +1,7 @@
+{:paths
+ ["src" "resources"]
+
+ :deps
+ {ai.djl/api {:mvn/version "0.36.0"}
+ ai.djl.huggingface/tokenizers {:mvn/version "0.36.0"}
+ ai.djl.onnxruntime/onnxruntime-engine {:mvn/version "0.36.0"}}}
diff --git a/modules/embedder/resources/metabase/embedder-model-catalog.edn b/modules/embedder/resources/metabase/embedder-model-catalog.edn
new file mode 100644
index 00000000000..c6ed61f986f
--- /dev/null
+++ b/modules/embedder/resources/metabase/embedder-model-catalog.edn
@@ -0,0 +1,39 @@
+{:catalog-version 1
+ :models
+ {"Snowflake/snowflake-arctic-embed-xs"
+ {:bundle-name "snowflake-arctic-embed-xs-d8c86521100d3556476a063fc2342036d45c106f"
+ :model-revision "d8c86521100d3556476a063fc2342036d45c106f"
+ :vector-dimensions 384
+ :runtime {:inference-contract-version 1
+ :djl-version "0.36.0"
+ :engine "OnnxRuntime"
+ :pooling "cls"
+ :normalize? true
+ :include-token-types? true}
+ :architectures {"arm64" {:source "onnx/model_int8.onnx"
+ :sha256 "e6aa5e656466a73d7c3111e9a3378bd13e5b93af30eaac2b3f13fd56692589a1"}
+ "avx2" {:source "onnx/model_uint8.onnx"
+ :sha256 "9295adb7774fe42f62fec11ad8cd07ed80c6b3ea112b2b737c683f562469b21b"}}
+ :tokenizer-files {"tokenizer.json" "91f1def9b9391fdabe028cd3f3fcc4efd34e5d1f08c3bf2de513ebb5911a1854"
+ "tokenizer_config.json" "9ca59277519f6e3692c8685e26b94d4afca2d5438deff66483db495e48735810"
+ "special_tokens_map.json" "5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a"
+ "config.json" "d7d071046ab952af96b7abad788db7ab3fc997b465e1b9914ff39707092254ec"}}
+
+ "sentence-transformers/all-MiniLM-L6-v2"
+ {:bundle-name "all-minilm-l6-v2-1110a243fdf4706b3f48f1d95db1a4f5529b4d41"
+ :model-revision "1110a243fdf4706b3f48f1d95db1a4f5529b4d41"
+ :vector-dimensions 384
+ :runtime {:inference-contract-version 1
+ :djl-version "0.36.0"
+ :engine "OnnxRuntime"
+ :pooling "mean"
+ :normalize? true
+ :include-token-types? true}
+ :architectures {"arm64" {:source "onnx/model_qint8_arm64.onnx"
+ :sha256 "4278337fd0ff3c68bfb6291042cad8ab363e1d9fbc43dcb499fe91c871902474"}
+ "avx2" {:source "onnx/model_quint8_avx2.onnx"
+ :sha256 "b941bf19f1f1283680f449fa6a7336bb5600bdcd5f84d10ddc5cd72218a0fd21"}}
+ :tokenizer-files {"tokenizer.json" "be50c3628f2bf5bb5e3a7f17b1f74611b2561a3a27eeab05e5aa30f411572037"
+ "tokenizer_config.json" "acb92769e8195aabd29b7b2137a9e6d6e25c476a4f15aa4355c233426c61576b"
+ "special_tokens_map.json" "303df45a03609e4ead04bc3dc1536d0ab19b5358db685b6f3da123d05ec200e3"
+ "config.json" "953f9c0d463486b10a6871cc2fd59f223b2c70184f49815e7efbcab5d8908b41"}}}}
diff --git a/modules/embedder/resources/metabase/embedder/metabase-plugin.yaml b/modules/embedder/resources/metabase/embedder/metabase-plugin.yaml
new file mode 100644
index 00000000000..85ed2e19572
--- /dev/null
+++ b/modules/embedder/resources/metabase/embedder/metabase-plugin.yaml
@@ -0,0 +1,8 @@
+info:
+ name: Metabase In-Process Embedder
+ version: 1.0.0
+ description: Runs a bundled text-embedding model inside the Metabase JVM.
+metabase-plugin-api-version: 1
+init:
+ - step: load-namespace
+ namespace: metabase-enterprise.embedder.plugin
diff --git a/modules/embedder/src/metabase_enterprise/embedder/catalog.clj b/modules/embedder/src/metabase_enterprise/embedder/catalog.clj
new file mode 100644
index 00000000000..9f170411fe9
--- /dev/null
+++ b/modules/embedder/src/metabase_enterprise/embedder/catalog.clj
@@ -0,0 +1,209 @@
+(ns metabase-enterprise.embedder.catalog
+ "Immutable metadata for the model artifact bundled in the embedder plugin."
+ (:require
+ [clojure.edn :as edn]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase.classloader.core :as classloader]
+ [metabase.util :as u])
+ (:import
+ (com.sun.jna Function NativeLibrary)
+ (java.nio.charset StandardCharsets)
+ (java.security MessageDigest)
+ (java.util HexFormat)))
+
+(set! *warn-on-reflection* true)
+
+(def default-model-name
+ "Canonical Library retrieval model bundled by this plugin."
+ "Snowflake/snowflake-arctic-embed-xs")
+
+(defn- plugin-resource
+ [resource-path]
+ (io/resource resource-path (classloader/the-classloader)))
+
+(def ^:private catalog
+ "Parsed model catalog bundled with the plugin."
+ (delay
+ (some-> "metabase/embedder-model-catalog.edn"
+ plugin-resource
+ slurp
+ edn/read-string)))
+
+(defn architecture
+ "Bundle architecture for this JVM. The x86 bundle uses the model's unsigned INT8 export."
+ []
+ (case (System/getProperty "os.arch")
+ ("aarch64" "arm64") "arm64"
+ ("amd64" "x86_64") "avx2"
+ nil))
+
+(defn supported-architecture?
+ "Whether this JVM architecture has a bundled model export."
+ []
+ (some? (architecture)))
+
+(defn- operating-system
+ []
+ (u/lower-case-en (System/getProperty "os.name")))
+
+(defn- process-maps
+ []
+ (try
+ (slurp "/proc/self/maps")
+ (catch Exception _
+ nil)))
+
+(defn- detect-linux-libc
+ [alpine? maps]
+ (cond
+ alpine?
+ :musl
+
+ (nil? maps)
+ :unknown
+
+ (str/includes? maps "ld-musl-")
+ :musl
+
+ (or (re-find #"/ld-(?:linux[^\s]*|\d+(?:\.\d+)*)\.so" maps)
+ (str/includes? maps "/libc.so.6"))
+ :glibc
+
+ :else
+ :unknown))
+
+(defn- linux-libc
+ []
+ (detect-linux-libc (.exists (io/file "/etc/alpine-release")) (process-maps)))
+
+(def ^:private minimum-glibc-version
+ "Minimum symbol version required by the pinned DJL 0.36 tokenizer natives."
+ [2 34])
+
+(defn- glibc-version
+ []
+ (try
+ (let [^NativeLibrary libc (NativeLibrary/getInstance "c")
+ ^Function function (.getFunction libc "gnu_get_libc_version")]
+ (.invokeString function (object-array 0) false))
+ (catch Throwable _
+ nil)))
+
+(defn- supported-glibc-version?
+ [version]
+ (boolean
+ (when-let [[_ major minor] (some->> version (re-matches #"^(\d+)\.(\d+).*$"))]
+ (not (neg? (compare [(parse-long major) (parse-long minor)] minimum-glibc-version))))))
+
+(defn runtime-readiness
+ "Whether the bundled DJL/ONNX native runtime can load on this JVM platform."
+ []
+ (let [os-name (operating-system)
+ arch (architecture)
+ glibc-version-string (when (= "linux" os-name) (glibc-version))
+ detected-libc (when (= "linux" os-name) (linux-libc))
+ libc (cond
+ ;; Keep explicit Alpine/musl evidence fail-closed even if a compatibility shim exports
+ ;; the GNU version symbol. Otherwise the API probe is stronger than an inconclusive map.
+ (= :musl detected-libc) :musl
+ (some? glibc-version-string) :glibc
+ :else detected-libc)]
+ (cond
+ (not (or (= "linux" os-name) (= "mac os x" os-name)))
+ {:ready? false :reason :unsupported-os}
+
+ (nil? arch)
+ {:ready? false :reason :unsupported-architecture}
+
+ ;; ONNX Runtime includes osx-x64, but the pinned DJL tokenizer ships only an osx-aarch64 native.
+ (and (= "mac os x" os-name) (= "avx2" arch))
+ {:ready? false :reason :unsupported-platform}
+
+ (and (= "linux" os-name) (not= :glibc libc))
+ {:ready? false :reason :unsupported-libc}
+
+ (and (= "linux" os-name) (not (supported-glibc-version? glibc-version-string)))
+ {:ready? false :reason :unsupported-libc-version}
+
+ :else
+ {:ready? true})))
+
+(defn model-spec
+ "Catalog entry for `model-name`, or nil when the plugin does not provide it."
+ [model-name]
+ (get-in @catalog [:models model-name]))
+
+(defn bundle-resource
+ "Classpath path of the bundle for `model-name` on this JVM."
+ [model-name]
+ (when-let [{:keys [bundle-name]} (and (architecture) (model-spec model-name))]
+ (str "metabase-embedder/" bundle-name "-" (architecture) ".zip")))
+
+(defn bundle-present?
+ "Whether the current plugin artifact contains the architecture-specific bundle for `model-name`."
+ [model-name]
+ (boolean (some-> model-name bundle-resource plugin-resource)))
+
+(defn- sha256
+ [^String value]
+ (let [^MessageDigest digest (MessageDigest/getInstance "SHA-256")
+ ^bytes bytes (.digest digest (.getBytes value StandardCharsets/UTF_8))]
+ (.formatHex (HexFormat/of) bytes)))
+
+(defn- stable-pr-str
+ [value]
+ (binding [*print-dup* false
+ *print-length* nil
+ *print-level* nil
+ *print-readably* true]
+ (pr-str value)))
+
+(defn resolved-model
+ "Resolve a supported requested model to the exact bundled vector space for this architecture."
+ [{:keys [provider model-name vector-dimensions]}]
+ (let [model-name (or model-name default-model-name)
+ arch (architecture)
+ runtime (runtime-readiness)
+ spec (model-spec model-name)]
+ (when-not (:ready? runtime)
+ (throw (ex-info (format "The in-process embedder native runtime does not support %s/%s (%s)."
+ (System/getProperty "os.name")
+ (System/getProperty "os.arch")
+ (name (:reason runtime)))
+ {:provider provider
+ :os (System/getProperty "os.name")
+ :architecture (System/getProperty "os.arch")
+ :reason (:reason runtime)})))
+ (when-not spec
+ (throw (ex-info (format "The in-process embedder does not bundle model %s." (pr-str model-name))
+ {:provider provider
+ :model-name model-name
+ :reason :model-not-bundled})))
+ (when (and vector-dimensions (not= vector-dimensions (:vector-dimensions spec)))
+ (throw (ex-info (format "The bundled model %s has %d dimensions, not %d."
+ (pr-str model-name) (:vector-dimensions spec) vector-dimensions)
+ {:provider provider
+ :model-name model-name
+ :configured-dimensions vector-dimensions
+ :vector-dimensions (:vector-dimensions spec)
+ :reason :vector-dimensions-mismatch})))
+ (let [arch-sha256 (get-in spec [:architectures arch :sha256])]
+ (when-not arch-sha256
+ (throw (ex-info (format "The bundled model %s has no artifact for architecture %s." (pr-str model-name) arch)
+ {:provider provider
+ :model-name model-name
+ :architecture arch
+ :reason :architecture-not-bundled}))))
+ (let [identity-input [:metabase-embedding-space-v1
+ model-name
+ (:model-revision spec)
+ (:vector-dimensions spec)
+ (into (sorted-map) (:runtime spec))
+ arch
+ (get-in spec [:architectures arch :sha256])
+ (sort (:tokenizer-files spec))]]
+ {:provider provider
+ :model-name model-name
+ :vector-dimensions (:vector-dimensions spec)
+ :embedding-space-id (str "emb:v1:sha256:" (sha256 (stable-pr-str identity-input)))})))
diff --git a/modules/embedder/src/metabase_enterprise/embedder/model.clj b/modules/embedder/src/metabase_enterprise/embedder/model.clj
new file mode 100644
index 00000000000..8194a68526c
--- /dev/null
+++ b/modules/embedder/src/metabase_enterprise/embedder/model.clj
@@ -0,0 +1,64 @@
+(ns metabase-enterprise.embedder.model
+ "Lazy DJL/ONNX Runtime lifecycle for the bundled embedding models."
+ (:require
+ [metabase-enterprise.embedder.catalog :as catalog]
+ [metabase.util.log :as log])
+ (:import
+ (ai.djl.huggingface.translator TextEmbeddingTranslatorFactory)
+ (ai.djl.inference Predictor)
+ (ai.djl.repository.zoo Criteria ZooModel)
+ (java.util ArrayList)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ^Class floats-class
+ (Class/forName "[F"))
+
+(defn- build-model
+ ^ZooModel [model-name]
+ (let [spec (catalog/model-spec model-name)
+ resource-path (catalog/bundle-resource model-name)]
+ (when-not (catalog/bundle-present? model-name)
+ (throw (ex-info (format "The embedder plugin does not contain the %s model bundle." model-name)
+ {:model-name model-name
+ :resource resource-path
+ :reason :model-bundle-missing})))
+ (log/info "Loading bundled in-process embedding model" model-name)
+ (-> (Criteria/builder)
+ (.setTypes String floats-class)
+ (.optEngine (get-in spec [:runtime :engine]))
+ (.optTranslatorFactory (TextEmbeddingTranslatorFactory.))
+ (.optModelName "model")
+ (.optModelUrls (str "jar:///" resource-path))
+ (.optArgument "pooling" (get-in spec [:runtime :pooling]))
+ (.optArgument "normalize" (str (boolean (get-in spec [:runtime :normalize?]))))
+ (.optArgument "includeTokenTypes" (str (boolean (get-in spec [:runtime :include-token-types?]))))
+ (.build)
+ (.loadModel))))
+
+;; Only successful loads are retained. A transient first-load failure is therefore retryable without restarting.
+;; Keep one resident ZooModel per catalog model: Library retrieval uses Arctic while Data Complexity Score uses MiniLM.
+(defonce ^:private loaded-models (atom {}))
+
+(defn- model
+ ^ZooModel [model-name]
+ (or (get @loaded-models model-name)
+ (locking loaded-models
+ (or (get @loaded-models model-name)
+ (let [loaded (build-model model-name)]
+ (swap! loaded-models assoc model-name loaded)
+ loaded)))))
+
+(defn reset-model!
+ "Close the resident model. Intended for tests and REPL use when no inference is active."
+ []
+ (locking loaded-models
+ (doseq [loaded (vals @loaded-models)]
+ (.close ^ZooModel loaded))
+ (reset! loaded-models {})))
+
+(defn embed-batch
+ "Embed one already-bounded batch of texts."
+ [model-name texts]
+ (with-open [predictor ^Predictor (.newPredictor (model model-name))]
+ (vec (.batchPredict predictor (ArrayList. ^java.util.Collection texts)))))
diff --git a/modules/embedder/src/metabase_enterprise/embedder/plugin.clj b/modules/embedder/src/metabase_enterprise/embedder/plugin.clj
new file mode 100644
index 00000000000..fb3f0cb9a6b
--- /dev/null
+++ b/modules/embedder/src/metabase_enterprise/embedder/plugin.clj
@@ -0,0 +1,81 @@
+(ns metabase-enterprise.embedder.plugin
+ "Manifest entry point for the in-process embedding provider."
+ (:require
+ [metabase-enterprise.embedder.catalog :as catalog]
+ [metabase.classloader.core :as classloader]
+ [metabase.embeddings.provider :as embeddings.provider]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(def provider-name
+ "Embedding provider ID registered by this plugin."
+ "in-process")
+
+(def max-batch-size
+ "Maximum number of texts passed to one padded local inference batch."
+ 32)
+
+;; This artifact contains the complete model and tokenizer, so DJL never needs network access. Set both DJL 0.36
+;; controls before loading any DJL class: offline mode prevents dependency/model downloads, and tracking opt-out
+;; prevents its default EC2 metadata probe and telemetry request.
+(System/setProperty "ai.djl.offline" "true")
+(System/setProperty "OPT_OUT_TRACKING" "true")
+
+(defn- readiness
+ [{:keys [model-name vector-dimensions]}]
+ (let [model-name (or model-name catalog/default-model-name)
+ runtime (catalog/runtime-readiness)
+ spec (catalog/model-spec model-name)]
+ (cond
+ (not (:ready? runtime))
+ runtime
+
+ (nil? spec)
+ {:ready? false :reason :model-not-bundled}
+
+ (and vector-dimensions (not= vector-dimensions (:vector-dimensions spec)))
+ {:ready? false :reason :vector-dimensions-mismatch}
+
+ (not (catalog/bundle-present? model-name))
+ {:ready? false :reason :model-bundle-missing}
+
+ :else
+ {:ready? true})))
+
+(defn- model-fn
+ [fn-name]
+ ;; The manifest loads only this registration namespace. Keep DJL classes and native initialization out of startup;
+ ;; resolve the implementation through the shared plugin classloader on first inference or explicit preparation.
+ ;; `require` skips its classloader setup when the namespace is already loaded, so install it unconditionally for
+ ;; callers running on thread pools that predate plugin initialization.
+ (classloader/the-classloader)
+ (classloader/require 'metabase-enterprise.embedder.model)
+ (or (ns-resolve 'metabase-enterprise.embedder.model fn-name)
+ (throw (ex-info (format "Embedder plugin model implementation does not define %s." fn-name)
+ {:provider provider-name :fn fn-name}))))
+
+(defn- embed-texts
+ [resolved-model texts _opts]
+ (if (empty? texts)
+ []
+ (let [embed-batch (model-fn 'embed-batch)]
+ (into []
+ (mapcat #(embed-batch (:model-name resolved-model) %))
+ (partition-all max-batch-size texts)))))
+
+(defn- prepare!
+ [resolved-model]
+ (let [start-ns (System/nanoTime)]
+ (embed-texts resolved-model ["warm-up probe"] {})
+ (let [elapsed-ms (quot (- (System/nanoTime) start-ns) 1000000)]
+ (log/info "In-process embedder warm-up took" elapsed-ms "ms")
+ elapsed-ms)))
+
+(embeddings.provider/register-provider!
+ provider-name
+ {:embedding-spi-version 1
+ :readiness readiness
+ :resolve-model catalog/resolved-model
+ :embed-texts embed-texts
+ :prepare! prepare!})
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index 4938b3b09b2..2e6abc43268 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -158,6 +158,12 @@
(catch Exception e
(log/warnf "Failed to register signal handler for SIG%s: %s" signal-name (ex-message e)))))))
+(def embedder-plugin-name
+ "Manifest `info.name` of the in-process embedder. Pinned against the manifest by
+ `metabase.embeddings.embedder-plugin-name-test`; a rename here without one there silently disables
+ the plugin, since `plugins/registered?` looks it up by this exact string."
+ "Metabase In-Process Embedder")
+
(defn- init!*
"General application initialization function which should be run once at application startup."
[]
@@ -178,6 +184,13 @@
(tracing/init!)
;; load any plugins as needed
(plugins/load-plugins!)
+ ;; The in-process embedder registers its provider during plugin initialization. Its model runtime remains
+ ;; lazy, so activating the lightweight registration namespace here does not load DJL or a model at startup.
+ (when (plugins/registered? embedder-plugin-name)
+ (try
+ (plugins/load-plugin! embedder-plugin-name)
+ (catch Exception e
+ (log/warnf "Unable to activate the in-process embedder plugin: %s" (ex-message e)))))
(init-status/set-progress! 0.3)
(setting/validate-settings-formatting!)
;; startup database. validates connection & runs any necessary migrations
diff --git a/src/metabase/embeddings/provider.clj b/src/metabase/embeddings/provider.clj
new file mode 100644
index 00000000000..ef64950a5f4
--- /dev/null
+++ b/src/metabase/embeddings/provider.clj
@@ -0,0 +1,251 @@
+(ns metabase.embeddings.provider
+ "Runtime registry and stable contract for embedding-provider plugins.
+
+ Plugin loading is capability-neutral and lives in `metabase.plugins`. This namespace owns only the embedding
+ contract: resolving a requested model to an immutable embedding space, reporting readiness, and producing vectors."
+ (:require
+ [clojure.string :as str]
+ [metabase.util.malli :as mu])
+ (:import
+ (java.nio.charset StandardCharsets)
+ (java.security MessageDigest)
+ (java.util HexFormat)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:const embedding-spi-version
+ "Current version of the embedding-provider registration contract. Provider plugins must embed this value at
+ build time rather than reading the host Var during initialization."
+ 1)
+
+(def ^:private implementation-schema
+ [:map
+ [:embedding-spi-version [:= embedding-spi-version]]
+ [:resolve-model fn?]
+ [:readiness fn?]
+ [:embed-texts fn?]
+ [:prepare! {:optional true} fn?]])
+
+(def ^:private resolved-model-schema
+ [:map {:closed true}
+ [:provider [:and :string [:fn (complement str/blank?)]]]
+ [:model-name [:and :string [:fn (complement str/blank?)]]]
+ [:vector-dimensions [:fn pos-int?]]
+ [:model-revision {:optional true} [:and :string [:fn (complement str/blank?)]]]
+ [:embedding-space-id [:and :string [:fn (complement str/blank?)]]]
+ [:embedding-spi-version [:= embedding-spi-version]]])
+
+(def ^:private readiness-schema
+ [:map {:closed true}
+ [:ready? :boolean]
+ [:reason {:optional true} :keyword]
+ [:message {:optional true} :string]])
+
+;; Re-registration is intentional: plugin namespaces can be reloaded during development without leaving a stale
+;; implementation in the registry.
+(defonce ^:private providers (atom {}))
+
+(defn register-provider!
+ "Register `implementation` under `provider-name`.
+
+ Implementations must declare the literal SPI version they were built against and provide three functions:
+
+ * `:resolve-model` turns a requested model map into a resolved descriptor with an immutable
+ `:embedding-space-id`.
+ * `:readiness` returns at least `{:ready? boolean}` and may include a keyword `:reason` and safe `:message`.
+ * `:embed-texts` accepts the resolved descriptor, a vector of texts, and an options map.
+
+ `:prepare!` is optional and accepts a resolved descriptor. Registration data must never contain credentials."
+ [provider-name implementation]
+ (when-not (and (string? provider-name) (not (str/blank? provider-name)))
+ (throw (ex-info "Embedding provider name must be a non-blank string."
+ {:provider provider-name})))
+ (when-not (= embedding-spi-version (:embedding-spi-version implementation))
+ (throw (ex-info (format "Embedding provider %s uses unsupported SPI version %s; this server supports %s."
+ (pr-str provider-name)
+ (pr-str (:embedding-spi-version implementation))
+ embedding-spi-version)
+ {:provider provider-name
+ :declared-spi-version (:embedding-spi-version implementation)
+ :supported-spi-version embedding-spi-version})))
+ (mu/validate-throw implementation-schema implementation)
+ (swap! providers assoc provider-name implementation)
+ provider-name)
+
+(defn registered?
+ "Whether `provider-name` has registered an embedding implementation."
+ [provider-name]
+ (contains? @providers provider-name))
+
+(defn registered-providers
+ "Names of all registered embedding providers."
+ []
+ (set (keys @providers)))
+
+(defn- provider-request
+ [{:keys [provider] :as requested-model}]
+ (when-let [requested-version (:embedding-spi-version requested-model)]
+ (when-not (= embedding-spi-version requested-version)
+ (throw (ex-info (format "Embedding model descriptor uses unsupported SPI version %s; this server supports %s."
+ (pr-str requested-version) embedding-spi-version)
+ {:provider provider
+ :declared-spi-version requested-version
+ :supported-spi-version embedding-spi-version}))))
+ ;; Space identity and SPI version are host-owned metadata on resolved descriptors, not provider request fields.
+ (dissoc requested-model :embedding-space-id :embedding-spi-version))
+
+(defn readiness
+ "Return structured readiness for `requested-model` without performing model resolution.
+
+ An absent plugin is represented as data rather than optimistic success, allowing callers to hide or disable
+ features until the provider is actually installed."
+ [{:keys [provider] :as requested-model}]
+ (if-let [implementation (get @providers provider)]
+ (let [result ((:readiness implementation) (provider-request requested-model))
+ public-result (select-keys result [:ready? :reason :message])]
+ (assoc (mu/validate-throw readiness-schema public-result) :provider provider))
+ {:provider provider
+ :ready? false
+ :reason :provider-not-registered}))
+
+(defn ready?
+ "Whether the provider for `requested-model` is installed and ready to embed."
+ [requested-model]
+ (true? (:ready? (readiness requested-model))))
+
+(defn- implementation!
+ [provider]
+ (or (get @providers provider)
+ (throw (ex-info (format "Embedding provider %s is not registered." (pr-str provider))
+ {:provider provider
+ :reason :provider-not-registered}))))
+
+(defn- resolve-model-with
+ [implementation {:keys [provider] :as requested-model}]
+ (let [resolved ((:resolve-model implementation) (provider-request requested-model))]
+ (when-not (= provider (:provider resolved))
+ (throw (ex-info "An embedding provider resolved a model for a different provider."
+ {:provider provider
+ :resolved-provider (:provider resolved)})))
+ (let [resolved (-> resolved
+ (select-keys [:provider :model-name :vector-dimensions :model-revision :embedding-space-id])
+ (assoc :embedding-spi-version embedding-spi-version))]
+ (mu/validate-throw resolved-model-schema resolved)
+ (when (and (:embedding-space-id requested-model)
+ (not= (:embedding-space-id requested-model) (:embedding-space-id resolved)))
+ (throw (ex-info "The requested embedding space no longer matches the provider's resolved model."
+ {:type ::embedding-space-changed
+ :provider provider
+ :model-name (:model-name resolved)
+ :requested-embedding-space-id (:embedding-space-id requested-model)
+ :resolved-embedding-space-id (:embedding-space-id resolved)})))
+ resolved)))
+
+(defn resolve-model
+ "Resolve a requested model to the public descriptor that identifies its exact embedding space.
+
+ The returned map is deliberately closed and contains no source locations, access tokens, or other provider-private
+ configuration. Consumers should persist `:embedding-space-id` anywhere vector compatibility matters."
+ [{:keys [provider] :as requested-model}]
+ (resolve-model-with (implementation! provider) requested-model))
+
+(defn- sha256
+ [^String value]
+ (let [^MessageDigest digest (MessageDigest/getInstance "SHA-256")
+ ^bytes bytes (.digest digest (.getBytes value StandardCharsets/UTF_8))]
+ (.formatHex (HexFormat/of) bytes)))
+
+(defn- stable-pr-str
+ [value]
+ ;; Embedding identity must not depend on request-thread printer bindings.
+ (binding [*print-dup* false
+ *print-length* nil
+ *print-level* nil
+ *print-readably* true]
+ (pr-str value)))
+
+(defn legacy-resolved-model
+ "Resolve a provider/model/dimensions tuple whose remote implementation has no stronger revision identity.
+
+ The ID deliberately covers only the vector-space contract and excludes endpoints, credentials, and other transport
+ configuration. Providers that can identify an exact model/tokenizer/export revision should supply their own ID."
+ [{:keys [provider model-name vector-dimensions model-revision] :as requested-model}]
+ (let [public-model (select-keys requested-model [:provider :model-name :vector-dimensions :model-revision])]
+ (assoc public-model
+ :embedding-space-id
+ (str "emb:v1:sha256:"
+ (sha256 (stable-pr-str [provider model-name vector-dimensions
+ (or model-revision :unspecified)]))))))
+
+(defn- ordered-values?
+ [value]
+ (or (sequential? value)
+ (instance? java.util.List value)
+ (and (some? value) (.isArray ^Class (class value)))))
+
+(defn embed-texts
+ "Embed `texts` with the provider selected by `requested-model`.
+
+ Model resolution happens before every call so the implementation always receives the canonical public descriptor.
+ The result must contain one vector per input, in the same order, and every vector must match the resolved dimensions."
+ ([requested-model texts]
+ (embed-texts requested-model texts {}))
+ ([{:keys [provider] :as requested-model} texts opts]
+ (let [implementation (implementation! provider)
+ resolved (resolve-model-with implementation requested-model)
+ texts (vec texts)
+ raw-embeddings ((:embed-texts implementation) resolved texts opts)]
+ (when-not (ordered-values? raw-embeddings)
+ (throw (ex-info "Embedding provider must return an ordered sequence or array of vectors."
+ {:provider provider
+ :actual-type (some-> raw-embeddings class .getName)})))
+ (let [embeddings (vec raw-embeddings)]
+ (when-not (= (count texts) (count embeddings))
+ (throw (ex-info "Embedding provider returned a different number of vectors than input texts."
+ {:provider provider
+ :expected-count (count texts)
+ :actual-count (count embeddings)})))
+ (doseq [[index embedding] (map-indexed vector embeddings)]
+ (when-not (ordered-values? embedding)
+ (throw (ex-info "Embedding vectors must be ordered sequences or arrays."
+ {:provider provider
+ :model-name (:model-name resolved)
+ :embedding-space-id (:embedding-space-id resolved)
+ :vector-index index
+ :actual-type (some-> embedding class .getName)})))
+ (when-not (= (:vector-dimensions resolved) (count embedding))
+ (throw (ex-info "Embedding vector dimensions do not match the resolved model."
+ {:provider provider
+ :model-name (:model-name resolved)
+ :embedding-space-id (:embedding-space-id resolved)
+ :vector-index index
+ :expected-dimensions (:vector-dimensions resolved)
+ :actual-dimensions (count embedding)})))
+ (when-let [[component-index invalid-value]
+ (first (keep-indexed (fn [component-index value]
+ (when-not (and (number? value)
+ (Double/isFinite (double value)))
+ [component-index value]))
+ embedding))]
+ (throw (ex-info "Embedding vectors must contain only finite numeric components."
+ {:provider provider
+ :model-name (:model-name resolved)
+ :embedding-space-id (:embedding-space-id resolved)
+ :vector-index index
+ :component-index component-index
+ :invalid-value invalid-value}))))
+ embeddings))))
+
+(defn embed-text
+ "Embed one text and return its vector."
+ ([requested-model text]
+ (embed-text requested-model text {}))
+ ([requested-model text opts]
+ (first (embed-texts requested-model [text] opts))))
+
+(defn prepare!
+ "Allow a provider to eagerly prepare the resolved model, for example by warming a lazy native runtime."
+ [requested-model]
+ (let [implementation (implementation! (:provider requested-model))]
+ (when-let [prepare-fn (:prepare! implementation)]
+ (prepare-fn (resolve-model-with implementation requested-model)))))
diff --git a/src/metabase/plugins/core.clj b/src/metabase/plugins/core.clj
index 9301081eda8..a61d27300eb 100644
--- a/src/metabase/plugins/core.clj
+++ b/src/metabase/plugins/core.clj
@@ -12,4 +12,5 @@
plugins-dir
plugins-dir-info]
[metabase.plugins.initialize
- load-plugin!])
+ load-plugin!
+ registered?])
diff --git a/src/metabase/plugins/initialize.clj b/src/metabase/plugins/initialize.clj
index d9ff8b24110..edef0b81ad5 100644
--- a/src/metabase/plugins/initialize.clj
+++ b/src/metabase/plugins/initialize.clj
@@ -77,6 +77,12 @@
:when info]
plugin-name))
+(defn registered?
+ "Whether a plugin with `plugin-name` has been registered."
+ [plugin-name]
+ {:pre [(string? plugin-name)]}
+ (some? (registered-info plugin-name)))
+
(defn- loaded? [plugin-name]
(boolean (get-in @plugins [plugin-name :loaded?])))
@@ -169,8 +175,8 @@
(register! plugin-info)))
:ok))
-(defn- registered? [{{plugin-name :name} :info}]
- (some? (registered-info plugin-name)))
+(defn- info-registered? [{{plugin-name :name} :info}]
+ (registered? plugin-name))
(mu/defn register-plugin-with-info!
"Register a plugin using parsed info from its manifest. Returns truthy if the plugin was successfully registered;
@@ -181,11 +187,11 @@
[:name :string]
[:version :string]]]]]
(validate-plugin-api-version! info)
- (or (registered? info)
+ (or (info-registered? info)
(do
(.lock plugin-lock)
(try
- (or (registered? info)
+ (or (info-registered? info)
(register! info))
(finally
(.unlock plugin-lock))))))
diff --git a/test/metabase/embeddings/embedder_plugin_artifact_test.clj b/test/metabase/embeddings/embedder_plugin_artifact_test.clj
new file mode 100644
index 00000000000..3ccbf87a56b
--- /dev/null
+++ b/test/metabase/embeddings/embedder_plugin_artifact_test.clj
@@ -0,0 +1,230 @@
+(ns metabase.embeddings.embedder-plugin-artifact-test
+ (:require
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.core.core :as core]
+ [metabase.embeddings.provider :as embeddings.provider]
+ [metabase.plugins.impl :as plugins]
+ [metabase.plugins.initialize :as plugin-initialize])
+ (:import
+ (java.net Proxy ProxySelector)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private requested-model
+ {:provider "in-process"
+ :model-name "Snowflake/snowflake-arctic-embed-xs"})
+
+(def ^:private requested-minilm-model
+ {:provider "in-process"
+ :model-name "sentence-transformers/all-MiniLM-L6-v2"})
+
+(defn- cosine
+ [^floats a ^floats b]
+ (loop [index 0, dot 0.0]
+ (if (< index (alength a))
+ (recur (inc index) (+ dot (* (aget a index) (aget b index))))
+ dot)))
+
+(defn- magnitude
+ [^floats embedding]
+ (Math/sqrt (cosine embedding embedding)))
+
+(defn- isolated-thread-call
+ [f]
+ (let [result (promise)
+ thread (Thread.
+ ^Runnable
+ (fn []
+ (try
+ (deliver result {:value (f)})
+ (catch Throwable e
+ (deliver result {:error e})))))]
+ ;; Simulate a worker created before plugin initialization, whose inherited loader cannot see the plugin JAR.
+ (.setContextClassLoader thread (ClassLoader/getPlatformClassLoader))
+ {:result result, :thread thread}))
+
+(defn- await-thread!
+ [{:keys [result thread]}]
+ (.start ^Thread thread)
+ (.join ^Thread thread 300000)
+ (when (.isAlive ^Thread thread)
+ (.interrupt ^Thread thread)
+ (throw (ex-info "Timed out waiting for isolated plugin test thread." {})))
+ (let [{:keys [value error]} @result]
+ (if error
+ (throw error)
+ value)))
+
+(defn- recording-proxy-selector
+ [attempts]
+ (proxy [ProxySelector] []
+ (select [uri]
+ (swap! attempts conj uri)
+ (java.util.Collections/singletonList Proxy/NO_PROXY))
+ (connectFailed [_uri _socket-address _exception])))
+
+(defn- check-missing-architecture!
+ "A catalog entry with no export for this architecture must fail rather than hash a nil sha256 into a
+ well-formed but meaningless embedding-space id."
+ [requested-model]
+ (let [model-spec-var (ns-resolve 'metabase-enterprise.embedder.catalog 'model-spec)
+ arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)
+ spec ((var-get model-spec-var) (:model-name requested-model))
+ without-arch (update spec :architectures dissoc ((var-get arch-var)))]
+ (with-redefs-fn {model-spec-var (constantly without-arch)}
+ ;; Assert the :reason, not just the message: callers branch on it.
+ #(is (= :architecture-not-bundled
+ (:reason (ex-data (try (embeddings.provider/resolve-model requested-model)
+ (catch clojure.lang.ExceptionInfo e e)))))
+ "a catalog entry missing this architecture's export fails instead of hashing nil"))))
+
+(deftest ^:sequential plugin-artifact-smoke-test
+ (if-not (= "true" (System/getenv "MB_EMBEDDER_ARTIFACT_TEST"))
+ (testing "artifact smoke is enabled only in its dedicated CI process"
+ (is true))
+ (let [pre-plugin-thread
+ (isolated-thread-call
+ #(hash-map :readiness (embeddings.provider/readiness requested-model)
+ :resolved (embeddings.provider/resolve-model requested-model)))]
+ (plugins/load-plugins!)
+ ;; Non-driver plugins are registered eagerly but initialized only when a feature selects them. The provider
+ ;; registration lives in the plugin init namespace, so activate this artifact explicitly before exercising it.
+ (plugin-initialize/load-plugin! core/embedder-plugin-name)
+ (testing "the implementation was activated through its jar manifest"
+ (is (embeddings.provider/registered? "in-process"))
+ (is (= "true" (System/getProperty "ai.djl.offline")))
+ (is (= "true" (System/getProperty "OPT_OUT_TRACKING")))
+ (is (str/starts-with? (str (io/resource "metabase_enterprise/embedder/plugin.clj")) "jar:file:"))
+ (is (nil? (find-ns 'metabase-enterprise.embedder.model))
+ "manifest registration must not initialize the DJL model namespace")
+ (let [{:keys [readiness resolved]} (await-thread! pre-plugin-thread)]
+ (is (true? (:ready? readiness)))
+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}" (:embedding-space-id resolved)))
+ (is (nil? (find-ns 'metabase-enterprise.embedder.model))
+ "catalog access from an old worker thread remains lazy")))
+ (testing "the built artifact contains a ready, immutable model space"
+ (is (true? (:ready? (embeddings.provider/readiness requested-model))))
+ (is (= {:provider "in-process" :ready? false :reason :vector-dimensions-mismatch}
+ (embeddings.provider/readiness
+ (assoc requested-model :vector-dimensions 768))))
+ (is (re-matches #"emb:v1:sha256:[0-9a-f]{64}"
+ (:embedding-space-id (embeddings.provider/resolve-model requested-model))))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"has 384 dimensions"
+ (embeddings.provider/resolve-model
+ (assoc requested-model :vector-dimensions 768))))
+ (let [architecture-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)
+ os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
+ libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)
+ glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)
+ resolve-for (fn [arch]
+ (with-redefs-fn {architecture-var (constantly arch)
+ os-var (constantly "linux")
+ libc-var (constantly :glibc)
+ glibc-version-var (constantly "2.34")}
+ #(embeddings.provider/resolve-model requested-model)))
+ normal (mapv (comp :embedding-space-id resolve-for) ["arm64" "avx2"])
+ bounded (binding [*print-length* 1
+ *print-level* 1
+ *print-readably* false
+ *print-dup* true]
+ (mapv (comp :embedding-space-id resolve-for) ["arm64" "avx2"]))]
+ (is (= normal bounded) "model identity ignores ambient printer bindings")
+ (is (apply not= normal)
+ "architecture-specific exports intentionally have distinct embedding spaces"))
+ (let [model-spec-var (ns-resolve 'metabase-enterprise.embedder.catalog 'model-spec)
+ model-spec-fn (var-get model-spec-var)
+ spec (model-spec-fn (:model-name requested-model))
+ normal (embeddings.provider/resolve-model requested-model)
+ reordered (update spec :runtime #(into (array-map) (reverse %)))
+ equivalent (with-redefs-fn {model-spec-var (constantly reordered)}
+ #(embeddings.provider/resolve-model requested-model))]
+ (is (= (:embedding-space-id normal) (:embedding-space-id equivalent))
+ "equivalent catalog map order does not change model identity")
+ (check-missing-architecture! requested-model))
+ (let [libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'linux-libc)
+ os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
+ detect-libc-var (ns-resolve 'metabase-enterprise.embedder.catalog 'detect-linux-libc)
+ detect-libc (var-get detect-libc-var)
+ glibc-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'glibc-version)
+ supported-version-var (ns-resolve 'metabase-enterprise.embedder.catalog 'supported-glibc-version?)
+ supported-version? (var-get supported-version-var)]
+ (is (= :unknown (detect-libc false nil)) "unreadable process maps fail closed")
+ (is (= :musl (detect-libc true nil)))
+ (is (= :musl (detect-libc false "/lib/ld-musl-x86_64.so.1")))
+ (is (= :glibc (detect-libc false "/usr/lib64/ld-2.28.so")))
+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2"))
+ "the Ubuntu 24.04 loader layout is recognized")
+ (is (= :glibc (detect-libc false "/usr/lib/x86_64-linux-gnu/libc.so.6")))
+ (is (false? (supported-version? "2.33")))
+ (is (true? (supported-version? "2.34")))
+ (is (true? (supported-version? "2.39")))
+ (is (false? (supported-version? nil)))
+ (with-redefs-fn {libc-var (constantly :glibc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.34")}
+ #(is (true? (:ready? (embeddings.provider/readiness requested-model)))))
+ (with-redefs-fn {libc-var (constantly :unknown)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.39")}
+ #(is (true? (:ready? (embeddings.provider/readiness requested-model)))
+ "the authoritative glibc API probe survives an inconclusive process map"))
+ (with-redefs-fn {libc-var (constantly :glibc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly "2.33")}
+ #(do
+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc-version}
+ (embeddings.provider/readiness requested-model)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc-version"
+ (embeddings.provider/resolve-model requested-model)))))
+ (doseq [[unsupported-libc probed-glibc-version] [[:musl "2.39"]
+ [:unknown nil]]]
+ (with-redefs-fn {libc-var (constantly unsupported-libc)
+ os-var (constantly "linux")
+ glibc-version-var (constantly probed-glibc-version)}
+ #(do
+ (is (= {:provider "in-process" :ready? false :reason :unsupported-libc}
+ (embeddings.provider/readiness requested-model)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unsupported-libc"
+ (embeddings.provider/resolve-model requested-model)))))))
+ (let [os-var (ns-resolve 'metabase-enterprise.embedder.catalog 'operating-system)
+ arch-var (ns-resolve 'metabase-enterprise.embedder.catalog 'architecture)]
+ (with-redefs-fn {os-var (constantly "mac os x")
+ arch-var (constantly "avx2")}
+ #(is (= {:provider "in-process" :ready? false :reason :unsupported-platform}
+ (embeddings.provider/readiness requested-model))
+ "the tokenizer artifact has no Intel macOS native library"))))
+ (testing "the artifact bounds local inference batches before loading the runtime"
+ (let [model-fn-var (ns-resolve 'metabase-enterprise.embedder.plugin 'model-fn)
+ batch-sizes (atom [])]
+ (with-redefs-fn
+ {model-fn-var (constantly
+ (fn [_model-name texts]
+ (swap! batch-sizes conj (count texts))
+ (mapv (fn [_] (float-array 384)) texts)))}
+ #(embeddings.provider/embed-texts requested-model (repeat 65 "text")))
+ (is (= [32 32 1] @batch-sizes))
+ (is (nil? (find-ns 'metabase-enterprise.embedder.model)))))
+ (testing "real inference runs through the plugin jar"
+ (let [network-attempts (atom [])
+ original-proxy (ProxySelector/getDefault)]
+ (try
+ (ProxySelector/setDefault (recording-proxy-selector network-attempts))
+ (let [[dog puppy invoice]
+ (await-thread!
+ (isolated-thread-call
+ #(embeddings.provider/embed-texts requested-model ["dog" "puppy" "invoice"])))
+ minilm-embeddings
+ (await-thread!
+ (isolated-thread-call
+ #(embeddings.provider/embed-texts requested-minilm-model ["dog" "invoice"])))]
+ (is (some? (find-ns 'metabase-enterprise.embedder.model)))
+ (is (= [384 384 384] (mapv alength [dog puppy invoice])))
+ (is (> (cosine dog puppy) (cosine dog invoice)))
+ (is (= [384 384] (mapv alength minilm-embeddings)))
+ (is (every? #(< (Math/abs (- 1.0 (magnitude %))) 1.0e-5) minilm-embeddings)
+ "MiniLM produces normalized 384-dimensional embeddings"))
+ (finally
+ (ProxySelector/setDefault original-proxy)))
+ (is (empty? @network-attempts) "bundled inference performs no outbound requests"))))))
diff --git a/test/metabase/embeddings/embedder_plugin_name_test.clj b/test/metabase/embeddings/embedder_plugin_name_test.clj
new file mode 100644
index 00000000000..6962c377463
--- /dev/null
+++ b/test/metabase/embeddings/embedder_plugin_name_test.clj
@@ -0,0 +1,14 @@
+(ns metabase.embeddings.embedder-plugin-name-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.core.core :as core]
+ [metabase.util.yaml :as yaml]))
+
+(set! *warn-on-reflection* true)
+
+(deftest embedder-plugin-name-matches-manifest-test
+ (testing "core/embedder-plugin-name is what plugins/registered? looks up by, so it must match the manifest"
+ (is (= (get-in (yaml/parse-string
+ (slurp "modules/embedder/resources/metabase/embedder/metabase-plugin.yaml"))
+ [:info :name])
+ core/embedder-plugin-name))))
diff --git a/test/metabase/embeddings/provider_test.clj b/test/metabase/embeddings/provider_test.clj
new file mode 100644
index 00000000000..832d0eb8a26
--- /dev/null
+++ b/test/metabase/embeddings/provider_test.clj
@@ -0,0 +1,290 @@
+(ns metabase.embeddings.provider-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.embeddings.provider :as embedding.provider]))
+
+(set! *warn-on-reflection* true)
+
+(defn- provider-name
+ []
+ (str "test-provider-" (random-uuid)))
+
+(defn- implementation
+ [calls & {:keys [readiness resolve-model embed-texts prepare!]
+ :or {readiness (constantly {:ready? true})
+ resolve-model embedding.provider/legacy-resolved-model
+ embed-texts (fn [model texts opts]
+ (swap! calls conj [:embed model texts opts])
+ (mapv (fn [_] (float-array (:vector-dimensions model))) texts))}}]
+ (cond-> {:embedding-spi-version embedding.provider/embedding-spi-version
+ :readiness readiness
+ :resolve-model resolve-model
+ :embed-texts embed-texts}
+ prepare! (assoc :prepare! prepare!)))
+
+(deftest registration-version-gate-test
+ (let [name (provider-name)]
+ (is (:const (meta #'embedding.provider/embedding-spi-version))
+ "AOT plugins must inline the SPI version they were compiled against")
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"unsupported SPI version"
+ (embedding.provider/register-provider!
+ name
+ {:embedding-spi-version (inc embedding.provider/embedding-spi-version)})))
+ (is (false? (embedding.provider/registered? name)))))
+
+(deftest missing-provider-readiness-test
+ (let [model {:provider (provider-name)}]
+ (is (= {:provider (:provider model)
+ :ready? false
+ :reason :provider-not-registered}
+ (embedding.provider/readiness model)))
+ (is (false? (embedding.provider/ready? model)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"is not registered"
+ (embedding.provider/resolve-model model)))))
+
+(deftest resolve-and-embed-test
+ (let [calls (atom [])
+ name (provider-name)
+ model {:provider name
+ :model-name "example/model"
+ :vector-dimensions 3}]
+ (embedding.provider/register-provider! name (implementation calls))
+ (is (embedding.provider/registered? name))
+ (is (contains? (embedding.provider/registered-providers) name))
+ (is (= {:provider name
+ :model-name "example/model"
+ :vector-dimensions 3
+ :embedding-space-id (:embedding-space-id (embedding.provider/legacy-resolved-model model))
+ :embedding-spi-version embedding.provider/embedding-spi-version}
+ (embedding.provider/resolve-model model)))
+ (let [embeddings (embedding.provider/embed-texts model ["one" "two"] {:purpose :document})]
+ (is (= [3 3] (mapv alength embeddings)))
+ (is (= [:embed
+ (embedding.provider/resolve-model model)
+ ["one" "two"]
+ {:purpose :document}]
+ (first @calls))))))
+
+(deftest resolved-model-is-public-and-stable-test
+ (let [name (provider-name)
+ calls (atom [])
+ requested {:provider name
+ :model-name "example/model"
+ :vector-dimensions 4
+ :api-key "do-not-leak"}]
+ (embedding.provider/register-provider! name (implementation calls))
+ (let [resolved (embedding.provider/resolve-model requested)]
+ (is (= #{:provider :model-name :vector-dimensions :embedding-space-id :embedding-spi-version}
+ (set (keys resolved))))
+ (is (not (re-find #"do-not-leak" (pr-str resolved))))
+ (is (= (:embedding-space-id resolved)
+ (:embedding-space-id (embedding.provider/resolve-model (dissoc requested :api-key)))))
+ (is (not= (:embedding-space-id resolved)
+ (:embedding-space-id
+ (embedding.provider/resolve-model (assoc requested :model-name "example/other")))))
+ (is (not= (:embedding-space-id resolved)
+ (:embedding-space-id
+ (embedding.provider/resolve-model (assoc requested :model-revision "revision-2"))))))
+ (testing "a persisted identity is checked against the provider's current resolution"
+ (let [resolved (embedding.provider/resolve-model requested)]
+ (is (= resolved (embedding.provider/resolve-model resolved)))
+ (try
+ (embedding.provider/resolve-model (assoc resolved :embedding-space-id "stale-space"))
+ (is false "expected a changed embedding space to be rejected")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :metabase.embeddings.provider/embedding-space-changed (:type (ex-data e))))))))
+ (testing "revisioned legacy descriptors round-trip without changing identity"
+ (let [revisioned (embedding.provider/resolve-model (assoc requested :model-revision "revision-1"))]
+ (is (= "revision-1" (:model-revision revisioned)))
+ (is (= revisioned (embedding.provider/resolve-model revisioned)))))
+ (testing "legacy identity is independent of ambient printer bindings"
+ (let [other-request (assoc requested :model-name "example/other")
+ normal (mapv (comp :embedding-space-id embedding.provider/resolve-model)
+ [requested other-request])
+ bounded (binding [*print-length* 1
+ *print-level* 1
+ *print-readably* false
+ *print-dup* true]
+ (mapv (comp :embedding-space-id embedding.provider/resolve-model)
+ [requested other-request]))]
+ (is (= normal bounded))
+ (is (apply not= bounded))))
+ (testing "legacy identity is pinned for JDBC integer dimensions"
+ (let [model {:provider "example" :model-name "example/model" :vector-dimensions (Integer/valueOf 4)}
+ expected "emb:v1:sha256:7f945085ff74c92a2daf851c4242084d0e955d7e0466e9b5a33e75aa188f68ee"]
+ (is (= expected (:embedding-space-id (embedding.provider/legacy-resolved-model model))))
+ (is (= expected
+ (binding [*print-dup* true]
+ (:embedding-space-id (embedding.provider/legacy-resolved-model model)))))))))
+
+(deftest resolved-descriptor-round-trip-strips-host-metadata-test
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}
+ inputs (atom [])]
+ (embedding.provider/register-provider!
+ name
+ (implementation
+ (atom [])
+ :resolve-model (fn [requested]
+ (swap! inputs conj requested)
+ (when (some #(contains? requested %) [:embedding-space-id :embedding-spi-version])
+ (throw (ex-info "closed provider request schema" {:request requested})))
+ (embedding.provider/legacy-resolved-model requested))))
+ (let [resolved (embedding.provider/resolve-model model)]
+ (is (= resolved (embedding.provider/resolve-model resolved)))
+ (is (= [model model] @inputs))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"unsupported SPI version"
+ (embedding.provider/resolve-model
+ (assoc resolved :embedding-spi-version
+ (inc embedding.provider/embedding-spi-version))))))))
+
+(deftest readiness-and-prepare-test
+ (let [name (provider-name)
+ prepared (atom nil)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom [])
+ :readiness (constantly {:ready? false :reason :runtime-unavailable :message "not ready"})
+ :prepare! #(reset! prepared %)))
+ (is (= {:provider name :ready? false :reason :runtime-unavailable :message "not ready"}
+ (embedding.provider/readiness model)))
+ (is (false? (embedding.provider/ready? model)))
+ (embedding.provider/prepare! model)
+ (is (= (embedding.provider/resolve-model model) @prepared))))
+
+(deftest readiness-validates-and-strips-host-metadata-test
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}
+ inputs (atom [])]
+ (embedding.provider/register-provider!
+ name
+ (implementation
+ (atom [])
+ :readiness (fn [requested]
+ (swap! inputs conj requested)
+ (when (some #(contains? requested %) [:embedding-space-id :embedding-spi-version])
+ (throw (ex-info "closed provider request schema" {:request requested})))
+ {:ready? true
+ :provider "provider-controlled"
+ :private-details {:api-key "do-not-leak"}})))
+ (let [resolved (embedding.provider/resolve-model model)
+ readiness (embedding.provider/readiness resolved)]
+ (is (= {:provider name :ready? true}
+ readiness)
+ "readiness exposes only the public result shape and the host-owned provider")
+ (is (not (re-find #"do-not-leak" (pr-str readiness))))
+ (is (= [model] @inputs))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"unsupported SPI version"
+ (embedding.provider/readiness
+ (assoc resolved :embedding-spi-version
+ (inc embedding.provider/embedding-spi-version)))))
+ (is (= [model] @inputs)
+ "the host rejects an unsupported descriptor before invoking the provider"))))
+
+(deftest provider-operation-uses-one-registry-snapshot-test
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}
+ implementation! (var-get #'embedding.provider/implementation!)
+ registry-lookups (atom 0)]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :prepare! (constantly nil)))
+ (with-redefs-fn {#'embedding.provider/implementation!
+ (fn [provider]
+ (swap! registry-lookups inc)
+ (implementation! provider))}
+ (fn []
+ (embedding.provider/embed-text model "one")
+ (is (= 1 @registry-lookups) "resolve and embed use one captured provider implementation")
+ (reset! registry-lookups 0)
+ (embedding.provider/prepare! model)
+ (is (= 1 @registry-lookups) "resolve and prepare use one captured provider implementation")))))
+
+(deftest invalid-provider-output-test
+ (testing "resolved model must describe the provider that registered it"
+ (let [name (provider-name)]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :resolve-model #(assoc % :provider "somebody-else"
+ :embedding-space-id "space")))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"different provider"
+ (embedding.provider/resolve-model
+ {:provider name :model-name "model" :vector-dimensions 2})))))
+ (testing "one vector is required per input"
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] [(float-array 2)])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"different number of vectors"
+ (embedding.provider/embed-texts model ["one" "two"])))))
+ (testing "every vector must match the resolved dimensions"
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] [(float-array 3)])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"dimensions do not match"
+ (embedding.provider/embed-text model "one")))))
+ (testing "every vector component must be finite and numeric"
+ (doseq [bad ["not-a-number" Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY]]
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] [[0.0 bad]])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"finite numeric components"
+ (embedding.provider/embed-text model "one"))
+ (pr-str bad))))))
+
+(deftest ordered-provider-output-test
+ (doseq [[description result]
+ [["vectors" [[0.0 1.0] [2.0 3.0]]]
+ ["lists" (list (list 0.0 1.0) (list 2.0 3.0))]
+ ["lazy sequences" (map #(map identity %) [[0.0 1.0] [2.0 3.0]])]
+ ["Java lists" (doto (java.util.ArrayList.)
+ (.addAll ^java.util.Collection [[0.0 1.0] [2.0 3.0]]))]
+ ["JVM arrays" (object-array [(float-array [0.0 1.0])
+ (object-array [2.0 3.0])])]]]
+ (testing description
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] result)))
+ (is (= [[0.0 1.0] [2.0 3.0]]
+ (mapv vec (embedding.provider/embed-texts model ["one" "two"]))))))))
+
+(deftest unordered-provider-output-test
+ (doseq [bad-result [#{[0.0 1.0] [2.0 3.0]}
+ {0.0 1.0, 2.0 3.0}
+ nil]]
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] bad-result)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"ordered sequence or array of vectors"
+ (embedding.provider/embed-texts model ["one" "two"]))
+ (pr-str bad-result))))
+ (doseq [bad-vector [#{0.0 1.0}
+ {0.0 1.0, 2.0 3.0}]]
+ (let [name (provider-name)
+ model {:provider name :model-name "model" :vector-dimensions 2}]
+ (embedding.provider/register-provider!
+ name
+ (implementation (atom []) :embed-texts (fn [_ _ _] [bad-vector])))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"vectors must be ordered sequences or arrays"
+ (embedding.provider/embed-text model "one"))
+ (pr-str bad-vector)))))
diff --git a/test/metabase/plugins/initialize_test.clj b/test/metabase/plugins/initialize_test.clj
index 92be18dcc76..69efaa6cec4 100644
--- a/test/metabase/plugins/initialize_test.clj
+++ b/test/metabase/plugins/initialize_test.clj
@@ -26,6 +26,7 @@
init-steps/do-init-steps! #(swap! calls conj [:init %])
lazy-loaded-driver/register-lazy-loaded-driver! #(swap! calls conj [:driver %])]
(is (= :ok (initialize/register-plugin-with-info! plugin)))
+ (is (true? (plugins/registered? plugin-name)))
(is (empty? @calls) "registration does not load plugin code")
(is (= :ok (plugins/load-plugin! plugin-name)))
(is (= :ok (plugins/load-plugin! plugin-name)) "loading is idempotent"))
@@ -207,6 +208,7 @@
(deftest load-plugin-requires-registered-plugin-test
(let [plugin-name (str "test missing plugin " (random-uuid))]
+ (is (false? (plugins/registered? plugin-name)))
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"is not registered"
(plugins/load-plugin! plugin-name)))))