Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Require deployment URLs for environment embedding keys (#82044)

Chris Truter43b53788f14c
diff --git a/enterprise/backend/src/metabase_enterprise/search/semantic/embedding.clj b/enterprise/backend/src/metabase_enterprise/search/semantic/embedding.clj
index efd0eddcd4e..365c8d5e1dd 100644
--- a/enterprise/backend/src/metabase_enterprise/search/semantic/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/search/semantic/embedding.clj
@@ -327,12 +327,18 @@
(defn embedder-circuit-untrusted?
"Whether the breaker is enabled and not closed -- i.e. open or half-open, so it doesn't yet trust the
embedding service (open short-circuits calls; half-open is on a single trial).
- False when the breaker is disabled or the provider has no circuit endpoint."
+ False when the breaker is disabled or the provider has no circuit endpoint.
+ True when resolving the endpoint throws, so a misconfigured embedder counts as untrusted, not as an error.
+ An `InterruptedException` still propagates."
[]
(boolean
(and (semantic-settings/semantic-search-embedder-circuit-breaker-enabled)
- (when-let [state (embedder-circuit-state)]
- (not= :closed state)))))
+ (try
+ (when-let [state (embedder-circuit-state)]
+ (not= :closed state))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception _ true)))))
(def ^:private request-specific-statuses
"HTTP statuses caused by a particular input, rather than the embedding service as a whole."
@@ -572,48 +578,72 @@
(str/replace #"/+$" ""))))
(defn- embedding-service-resolve-config!
- "Return the embedding endpoint config, or throw if no base URL is configured.
-
- `:instance-token?` says the request authenticates with the instance token rather than with an API key of its own.
- The token is deployment credential, not a setting anyone can enter, so it only travels to an endpoint the
- deployment chose: `ee-embedding-service-base-url` earns it from the environment, not from the app DB.
-
- A base URL the environment supplies is deployment-controlled in the same way: its `:allow-private` policy floor
- admits private addresses but still blocks loopback and link-local."
+ "Return the embedding endpoint config as `{:endpoint :api-key :instance-token? :network-policy-floor}`.
+ Throw an ex-info with an `:error-code` when the settings cannot be used together.
+ Its message names the settings to change and never their values.
+
+ The `:instance-token?` flag says the request authenticates with the instance token, not its own API key.
+ The instance token only goes to an endpoint the environment chose, never to a URL stored in the app DB.
+ A base URL from the environment also gets the `:allow-private` policy floor.
+ That floor admits private addresses but still blocks loopback and link-local."
[]
- ;; the floor and the token are granted per source, not per setting: a settings manager can write the stored
- ;; ee-embedding-service-base-url through the generic settings API, while a value the environment supplies bypasses
- ;; the vetting setter and is trusted instead
- (cond (string? (not-empty (semantic-settings/ee-embedding-service-base-url)))
- (let [env-url? (some? (setting/env-var-value :ee-embedding-service-base-url))
- api-key (semantic-settings/ee-embedding-service-api-key)]
- (when-not (or env-url? (not-empty api-key))
- (throw (ex-info (str "The embedding service base URL is set in the application database and has no API "
- "key. Set " (setting/env-var-name :ee-embedding-service-base-url)
- " to use the instance token, or configure "
- (setting/env-var-name :ee-embedding-service-api-key) ".")
- {:settings ["ee-embedding-service-base-url"
+ ;; Trust follows where each value came from.
+ ;; A settings manager can store ee-embedding-service-base-url through the generic settings API.
+ ;; An environment value comes from the deployment and skips the vetting setter.
+ (let [base-url (not-empty (semantic-settings/ee-embedding-service-base-url))
+ ;; A whitespace-only environment URL still overrides the stored one, and normalizes to nil.
+ ;; Checking base-url first keeps it from counting as an environment URL.
+ env-url? (boolean (and base-url (setting/env-var-value :ee-embedding-service-base-url)))
+ env-key? (some? (setting/env-var-value :ee-embedding-service-api-key))]
+ ;; A key from the deployment only goes to a URL from the deployment.
+ ;; This holds even when the URL was saved through the API before the key was set.
+ (when (and env-key? (not env-url?))
+ (throw (ex-info (str "Set " (setting/env-var-name :ee-embedding-service-base-url)
+ " alongside " (setting/env-var-name :ee-embedding-service-api-key)
+ "; an environment API key is only sent to an embedding service URL"
+ " from the environment.")
+ {:error-code :embedding-environment-key-requires-environment-url
+ :settings ["ee-embedding-service-base-url"
+ "ee-embedding-service-api-key"]})))
+ (if base-url
+ (let [api-key (semantic-settings/ee-embedding-service-api-key)]
+ (when-not (or env-url? (not-empty api-key))
+ (throw (ex-info (str "The embedding service base URL is set in the application database"
+ " and has no API key. Set " (setting/env-var-name :ee-embedding-service-base-url)
+ " to use the instance token, or configure an API key"
+ " in the application database.")
+ {:error-code :embedding-database-url-requires-api-key
+ :settings ["ee-embedding-service-base-url"
"ee-embedding-service-api-key"]})))
- (cond-> {:endpoint (str (trim-trailing-slashes (semantic-settings/ee-embedding-service-base-url))
- "/v1/embeddings")
- :api-key api-key
- :instance-token? env-url?}
- env-url? (assoc :network-policy-floor :allow-private)))
-
- (string? (not-empty (llm.settings/ai-service-base-url)))
- (cond-> {:endpoint (str (trim-trailing-slashes (llm.settings/ai-service-base-url)) "/v1/embeddings")
+ (cond-> {:endpoint (str (trim-trailing-slashes base-url) "/v1/embeddings")
+ :api-key api-key
+ :instance-token? env-url?}
+ env-url? (assoc :network-policy-floor :allow-private)))
+ (if-let [ai-service-url (not-empty (llm.settings/ai-service-base-url))]
+ (cond-> {:endpoint (str (trim-trailing-slashes ai-service-url) "/v1/embeddings")
:instance-token? true}
- (setting/env-var-value :ai-service-base-url)
- (assoc :network-policy-floor :allow-private))
-
- :else
+ (setting/env-var-value :ai-service-base-url) (assoc :network-policy-floor :allow-private))
(throw (ex-info "Embedding service and ai service base URLs are not configured"
- {:settings ["ee-embedding-service-base-url"
- "ai-service-base-url"]}))))
+ {:error-code :embedding-service-url-not-configured
+ :settings ["ee-embedding-service-base-url"
+ "ai-service-base-url"]}))))))
(defmethod embedder-circuit-endpoint "ai-service" [_]
(:endpoint (embedding-service-resolve-config!)))
+(defn- ai-service-readiness
+ "Return `{:ready? true}` when [[embedding-service-resolve-config!]] accepts the current settings.
+ Otherwise return `{:ready? false}` with the refusal's `:message`, and any `:error-code` as `:reason`."
+ [_]
+ (try
+ (embedding-service-resolve-config!)
+ {:ready? true}
+ (catch clojure.lang.ExceptionInfo e
+ (let [reason (:error-code (ex-data e))]
+ (cond-> {:ready? false
+ :message (ex-message e)}
+ reason (assoc :reason reason))))))
+
(defn- ai-service-get-embeddings-batch
[{:keys [model-name vector-dimensions]} texts {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]
(let [{:keys [endpoint api-key network-policy-floor instance-token?]} (embedding-service-resolve-config!)]
@@ -691,9 +721,7 @@
(embeddings.provider/register-provider!
"ai-service"
{:embedding-spi-version spi-version
- :readiness (fn [_]
- {:ready? (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))
- (not-empty (llm.settings/ai-service-base-url))))})
+ :readiness ai-service-readiness
:resolve-model legacy
:embed-texts ai-service-get-embeddings-batch})
(embeddings.provider/register-provider!
diff --git a/enterprise/backend/src/metabase_enterprise/search/semantic/settings.clj b/enterprise/backend/src/metabase_enterprise/search/semantic/settings.clj
index 4836855dd51..35221123d55 100644
--- a/enterprise/backend/src/metabase_enterprise/search/semantic/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/search/semantic/settings.clj
@@ -96,8 +96,10 @@
:doc false)
(defsetting ee-embedding-service-api-key
- (deferred-tru (str "API key for authenticating with the embedding service. Leave empty for proxying thorugh"
- " ai-service. In that case premium-embedding-token is used for authentication."))
+ (deferred-tru (str "API key for authenticating with the embedding service. Leave empty for proxying through"
+ " ai-service. In that case premium-embedding-token is used for authentication."
+ " When the API key is set through an environment variable, set the embedding service"
+ " base URL through the environment too."))
:sensitive? true
:visibility :settings-manager
:export? false
diff --git a/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_circuit_breaker_test.clj
index 75b9421ff75..cdfe244b814 100644
--- a/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_circuit_breaker_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_circuit_breaker_test.clj
@@ -27,6 +27,16 @@
(defmethod semantic.embedding/embedder-circuit-endpoint test-provider [_] test-endpoint)
+(def ^:private misconfigured-provider ::misconfigured)
+
+(defmethod semantic.embedding/embedder-circuit-endpoint misconfigured-provider [_]
+ (throw (ex-info "embedding service misconfigured" {})))
+
+(def ^:private interrupted-provider ::interrupted)
+
+(defmethod semantic.embedding/embedder-circuit-endpoint interrupted-provider [_]
+ (throw (InterruptedException. "cancelled")))
+
(def ^:private call-through* #'semantic.embedding/call-through-embedder-breaker)
(def ^:private openai-compatible-get-embeddings-batch* #'semantic.embedding/openai-compatible-get-embeddings-batch)
(def ^:private validate-embeddings!* #'semantic.embedding/validate-embeddings!)
@@ -63,6 +73,20 @@
(is (= "https://openai.example/v1/embeddings"
(semantic.embedding/embedder-circuit-endpoint {:provider "openai"}))))))
+(deftest misconfigured-provider-counts-as-untrusted-test
+ (testing "a provider whose endpoint resolution throws is untrusted, not an error"
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly {:provider misconfigured-provider})]
+ (is (true? (semantic.embedding/embedder-circuit-untrusted?)))))))
+
+(deftest interruption-during-resolution-propagates-test
+ (testing "an interruption during endpoint resolution propagates, rather than counting as untrusted"
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly {:provider interrupted-provider})]
+ (is (thrown? InterruptedException (semantic.embedding/embedder-circuit-untrusted?)))))))
+
(deftest ^:synchronized request-failure-does-not-change-service-failure-history-test
(testing "request-, model-, and policy-specific failures neither trip the circuit nor reset service-failure history"
(let [breaker (dh.cb/circuit-breaker {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})]
diff --git a/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_test.clj
index 642f2390d48..eece39e49ef 100644
--- a/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/semantic/embedding_test.clj
@@ -605,6 +605,53 @@
(is (= "https://1.1.1.1" (semantic.settings/ee-embedding-service-base-url)))
(is (= "replacement-key" (semantic.settings/ee-embedding-service-api-key))))))))
+(deftest embedding-environment-key-refuses-a-previously-stored-url-test
+ (mt/with-premium-features #{:advanced-permissions}
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url nil
+ ee-embedding-service-api-key nil]
+ (mt/with-user-in-groups [group {:name "Embedding settings managers"}
+ user [group]]
+ (perms/grant-application-permissions! group :setting)
+ (is (nil? (mt/user-http-request user :put 204 "setting/ee-embedding-service-base-url"
+ {:value "https://1.1.1.1"})))
+ (mt/with-temp-env-var-value! [mb-ee-embedding-service-api-key "deployment-key"]
+ (let [requests (atom [])]
+ (mt/with-dynamic-fn-redefs [http/post (fn [& args] (swap! requests conj args))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Set MB_EE_EMBEDDING_SERVICE_BASE_URL alongside MB_EE_EMBEDDING_SERVICE_API_KEY"
+ (embedding/get-embedding {:provider "ai-service", :model-name "m", :vector-dimensions 3}
+ "text" {:record-tokens? false})))
+ (is (empty? @requests)))))))))
+
+(defn- resolved-endpoint-and-key!
+ "Return `[endpoint api-key]` from the resolved embedding-service config, or the refusal's `:error-code`."
+ []
+ (try
+ ((juxt :endpoint :api-key) (#'embedding/embedding-service-resolve-config!))
+ (catch clojure.lang.ExceptionInfo e
+ (:error-code (ex-data e)))))
+
+(deftest embedding-credential-source-combinations-test
+ (mt/with-dynamic-fn-redefs [llm.settings/ai-service-base-url (constantly "https://ai.test")]
+ (doseq [[stored-url env-url env-key expected]
+ [;; accepted: stored URL, environment URL, environment pair, then the ai-service fallback
+ ["https://8.8.8.8" nil nil ["https://8.8.8.8/v1/embeddings" "stored-key"]]
+ ["https://8.8.8.8" "https://1.1.1.1" nil ["https://1.1.1.1/v1/embeddings" "stored-key"]]
+ ["https://8.8.8.8" "https://1.1.1.1" "env-key" ["https://1.1.1.1/v1/embeddings" "env-key"]]
+ [nil nil nil ["https://ai.test/v1/embeddings" nil]]
+ ;; refused: an environment key with no usable environment URL, whether or not another URL is set
+ ["https://8.8.8.8" nil "env-key" :embedding-environment-key-requires-environment-url]
+ [nil nil "env-key" :embedding-environment-key-requires-environment-url]
+ ["https://8.8.8.8" " \t " "env-key" :embedding-environment-key-requires-environment-url]
+ ["https://8.8.8.8" nil " " :embedding-environment-key-requires-environment-url]]]
+ (testing (pr-str {:stored-url stored-url, :env-url env-url, :env-key env-key})
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url stored-url
+ ee-embedding-service-api-key "stored-key"]
+ (mt/with-temp-env-var-value! [mb-ee-embedding-service-base-url env-url
+ mb-ee-embedding-service-api-key env-key]
+ (is (= expected (resolved-endpoint-and-key!)))))))))
+
(deftest test-embedding-service-snowplow-tracking
(testing "ai-service fires a Snowplow token_usage event on each batch call"
(mt/with-temporary-setting-values [ee-embedding-service-base-url "http://mock-embedding-service"
@@ -747,16 +794,30 @@
(is (= "Bearer sk-embed" (get @captured "Authorization")))))))))
(deftest embedding-supported?-test
- (testing "ai-service: supported iff an embedding-service base URL or the ai-service base URL is set"
- (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://embed"]
+ (testing "ai-service: supported iff the embedding-service config resolves"
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://embed"
+ ee-embedding-service-api-key "stored-key"]
(is (true? (embedding/embedding-supported? {:provider "ai-service"}))))
+ (testing "a connection that would refuse to embed is not ready, and says why"
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://embed"
+ ee-embedding-service-api-key nil]
+ (is (=? {:ready? false
+ :reason :embedding-database-url-requires-api-key
+ :message #"(?s).*MB_EE_EMBEDDING_SERVICE_BASE_URL.*"}
+ (embeddings.provider/readiness {:provider "ai-service"})))
+ (mt/with-temp-env-var-value! [mb-ee-embedding-service-api-key "deployment-key"]
+ (is (=? {:ready? false
+ :reason :embedding-environment-key-requires-environment-url}
+ (embeddings.provider/readiness {:provider "ai-service"}))))))
;; ai-service-base-url is :enabled? only under the AI-managed / metabot-v3 features, so license one to
;; exercise that branch (and to make the setting settable/restorable here).
(mt/with-premium-features #{:metabot-v3}
(mt/with-temporary-setting-values [ee-embedding-service-base-url nil ai-service-base-url "http://ai"]
(is (true? (embedding/embedding-supported? {:provider "ai-service"})))))
(mt/with-temporary-setting-values [ee-embedding-service-base-url nil]
- (is (false? (embedding/embedding-supported? {:provider "ai-service"})))))
+ (is (=? {:ready? false
+ :reason :embedding-service-url-not-configured}
+ (embeddings.provider/readiness {:provider "ai-service"})))))
(testing "openai: supported iff the connection semantic search embeds through has an API key"
(llm.tu/with-connections [(llm.tu/connection "openai" {:api-key "sk-test"})]
(is (true? (embedding/embedding-supported? {:provider "openai"}))))