The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enforce a network policy on LLM provider base URLs (#81301)
Chris Truterf508214657ab
diff --git a/e2e/runner/cypress-runner-backend.js b/e2e/runner/cypress-runner-backend.jsindex ad92c856bcb..bc52c776870 100755--- a/e2e/runner/cypress-runner-backend.js+++ b/e2e/runner/cypress-runner-backend.js@@ -33,6 +33,8 @@ process.env.MB_SAMPLE_DATABASE_ENGINE = "h2";// add or sync a test database. Set here rather than in the `:e2e` deps.edn alias because CI runs the uberjar and// never reads that alias. A spec that wants the policy on can set it through the settings API.process.env.MB_WAREHOUSE_ALLOWED_NETWORKS = "allow-all";+// Metabot specs use a mock LLM server on localhost, so E2E runs must allow loopback LLM endpoints.+process.env.MB_LLM_ALLOWED_NETWORKS = "allow-all";if (!process.env.CI) {// Use a temporary copy of the sample db so it won't use and lock the db used for local developmentdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.cljindex aecd4f33aef..eb69e1510d8 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj@@ -13,6 +13,7 @@[metabase.embeddings.provider :as embeddings.provider][metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features]+ [metabase.settings.core :as setting][metabase.tracing.core :as tracing][metabase.util :as u][metabase.util.json :as json]@@ -335,12 +336,11 @@(defn- service-failure?[e]- (let [data (ex-data e)- cause (some-> e ex-cause ex-data)- status (or (:status data) (:status cause))- reason (or (:cause data) (:cause cause))]+ (let [{:keys [status cause]} (u/all-ex-data e)](and (not (contains? request-specific-statuses status))- (not= :embedder/unexpected-dimensions reason))))+ (not= :embedder/unexpected-dimensions cause)+ ;; A connection-time network-policy rejection is about the configured endpoint, not the service's health.+ (not (llm.settings/llm-network-policy-error? e)))))(defn- circuit-open-ex[endpoint]@@ -459,18 +459,21 @@"Call an OpenAI-compatible /v1/embeddings endpoint. The breaker guards only the remote request andresponse decoding; analytics and token persistence happen after it returns.- `:provider` — label for analytics (e.g. \"ai-service\", \"openai\")- `:endpoint` — full URL including /v1/embeddings- `:api-key` — Bearer token. If empty ai service proxying is assumed and premium-embedding-token is- used for authentication- `:model-name` — model identifier sent in the request body- `:vector-dimensions` — expected dimensions of each returned vector- `:texts` — collection of input strings- `:record-tokens?` — true writes a `semantic_search_token_tracking` row, false skips it.- `:snowplow?` — optional; when true fires a Snowplow `token_usage` event- `:extra-body` — optional; merged into the request body (e.g. `{:dimensions 1024}`)- `:type` — optional; forwarded to the token-tracking row"- [{:keys [provider endpoint api-key model-name vector-dimensions texts record-tokens? extra-body snowplow?]+ `:provider` - label for analytics (e.g. \"ai-service\", \"openai\")+ `:endpoint` - full URL including /v1/embeddings+ `:api-key` - Bearer token. For `ai-service`, an empty value uses `premium-embedding-token`.+ `:model-name` - model identifier sent in the request body+ `:vector-dimensions` - expected dimensions of each returned vector+ `:texts` - collection of input strings+ `:record-tokens?` - true writes a `semantic_search_token_tracking` row, false skips it.+ `:snowplow?` - optional; when true fires a Snowplow `token_usage` event+ `:extra-body` - optional; merged into the request body (e.g. `{:dimensions 1024}`)+ `:type` - optional; forwarded to the token-tracking row++ `:network-policy-floor` optionally sets the minimum network access for a deployment-controlled endpoint; see+ [[metabase.llm.settings/network-policy]]."+ [{:keys [provider endpoint api-key model-name vector-dimensions texts record-tokens? extra-body snowplow?+ network-policy-floor]:as opts}:- [:map[:provider :string]@@ -481,68 +484,74 @@[:texts [:sequential :string]][:record-tokens? :boolean][:snowplow? {:optional true} [:maybe :boolean]]- [:extra-body {:optional true} [:maybe :map]]]]- (try- (log/debug (str "Calling " provider " embeddings API")- {:endpoint endpoint :documents (count texts) :tokens (count-tokens-batch texts)})- (let [headers (merge {"Content-Type" "application/json"}- (if (and (empty? api-key) (= "ai-service" provider))- {"x-metabase-instance-token"- (u/prog1 (premium-features/premium-embedding-token)- (when (nil? <>)- (throw (ex-info "Premium embedding token not set"- {:provider provider}))))}- {"Authorization" (str "Bearer " api-key)}))- request (merge embedding-http-timeouts- {:headers headers- :body (json/encode- (merge {:model model-name- :input texts- :encoding_format "base64"}- extra-body))})- start-ms (u/start-timer)- {:keys [usage embeddings]}- (call-through-embedder-breaker- #(let [{:keys [usage data]} (-> (http/post endpoint request)- :body- (json/decode true))]- {:usage usage- :embeddings (validate-embeddings! (decode-embeddings data)- (count texts)- vector-dimensions)})- :endpoint endpoint)- total-tokens (:total_tokens usage 0)- prompt-tokens (:prompt_tokens usage total-tokens)]- (analytics/inc! :metabase-search/semantic-embedding-tokens- {:provider provider :model model-name}- total-tokens)- (when snowplow?- (analytics.core/track-token-usage!- {:snowplow true- :prometheus false ; already tracked via inc! above- :request-id (analytics.core/uuid->ai-service-hex-uuid (random-uuid))- :model-id model-name- :total-tokens total-tokens- :prompt-tokens prompt-tokens- :completion-tokens 0 ; embedding models don't produce completion tokens- :estimated-costs-usd 0.0- :duration-ms (long (u/since-ms start-ms))- :tag "embedding_generation"}))- (when record-tokens?- (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens))- embeddings)- (catch ConnectException e- (log/error (str "Failed to connect to " provider ": " (ex-message e)) {:endpoint endpoint})- (throw (ex-info (str provider " unavailable (connection refused)")- {:status 502 :endpoint endpoint}- e)))- (catch Exception e- ;; The breaker transition already logs the outage once. Fast-failed calls while it remains open are- ;; expected and can be frequent, so do not emit a redundant error for every guarded request.- (when-not (= :embedder/circuit-open (:cause (ex-data e)))- (log/error (str "Failed to generate " provider " embeddings: " (ex-message e))- {:documents (count texts) :tokens (count-tokens-batch texts)}))- (throw e))))+ [:extra-body {:optional true} [:maybe :map]]+ [:network-policy-floor {:optional true} [:maybe [:enum :external-only :allow-private :allow-all]]]]]+ ;; Outside the try: a malformed endpoint is neither a service failure nor something to log per batch.+ (let [policy-opts (llm.settings/llm-request-opts network-policy-floor endpoint)]+ (try+ (log/debug (str "Calling " provider " embeddings API")+ {:endpoint endpoint :documents (count texts) :tokens (count-tokens-batch texts)})+ (let [headers (merge {"Content-Type" "application/json"}+ (if (and (empty? api-key) (= "ai-service" provider))+ {"x-metabase-instance-token"+ (u/prog1 (premium-features/premium-embedding-token)+ (when (nil? <>)+ (throw (ex-info "Premium embedding token not set"+ {:provider provider}))))}+ {"Authorization" (str "Bearer " api-key)}))+ request (merge embedding-http-timeouts+ {:headers headers+ :body (json/encode+ (merge {:model model-name+ :input texts+ :encoding_format "base64"}+ extra-body))}+ policy-opts)+ start-ms (u/start-timer)+ {:keys [usage embeddings]}+ (call-through-embedder-breaker+ #(let [{:keys [usage data]} (-> (http/post endpoint request)+ :body+ (json/decode true))]+ {:usage usage+ :embeddings (validate-embeddings! (decode-embeddings data)+ (count texts)+ vector-dimensions)})+ :endpoint endpoint)+ total-tokens (:total_tokens usage 0)+ prompt-tokens (:prompt_tokens usage total-tokens)]+ (analytics/inc! :metabase-search/semantic-embedding-tokens+ {:provider provider :model model-name}+ total-tokens)+ (when snowplow?+ (analytics.core/track-token-usage!+ {:snowplow true+ :prometheus false ; already tracked via inc! above+ :request-id (analytics.core/uuid->ai-service-hex-uuid (random-uuid))+ :model-id model-name+ :total-tokens total-tokens+ :prompt-tokens prompt-tokens+ :completion-tokens 0 ; embedding models don't produce completion tokens+ :estimated-costs-usd 0.0+ :duration-ms (long (u/since-ms start-ms))+ :tag "embedding_generation"}))+ (when record-tokens?+ (semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens))+ embeddings)+ (catch ConnectException e+ (llm.settings/rethrow-if-llm-network-policy-error! e endpoint)+ (log/error (str "Failed to connect to " provider ": " (ex-message e)) {:endpoint endpoint})+ (throw (ex-info (str provider " unavailable (connection refused)")+ {:status 502 :endpoint endpoint}+ e)))+ (catch Exception e+ (llm.settings/rethrow-if-llm-network-policy-error! e endpoint)+ ;; The breaker transition already logs the outage once. Fast-failed calls while it remains open are+ ;; expected and can be frequent, so do not emit a redundant error for every guarded request.+ (when-not (= :embedder/circuit-open (:cause (ex-data e)))+ (log/error (str "Failed to generate " provider " embeddings: " (ex-message e))+ {:documents (count texts) :tokens (count-tokens-batch texts)}))+ (throw e)))));;;; Embedding-service provider@@ -553,17 +562,26 @@(str/replace #"/+$" ""))))(defn- embedding-service-resolve-config!- "Returns [endpoint api-key]. When api key is not set or when service url is not set but- `llm.settings/ai-service-base-url` is set the ai service proxying is assumed. In that case premium-embedding-token- is used for authentication. Throws if neither base URL is configured."+ "Return the embedding endpoint config, or throw if no base URL is configured.++ Requests without an API key use the premium embedding token. A base URL the environment supplies is+ deployment-controlled: its `:allow-private` policy floor admits private addresses but still blocks loopback and+ link-local."[]+ ;; the floor is granted per source, not per setting: a superuser can write either stored setting through the+ ;; generic settings API, which must not widen the policy, while a value the environment supplies bypasses the+ ;; vetting setter and is trusted instead(cond (string? (not-empty (semantic-settings/ee-embedding-service-base-url)))- [(str (trim-trailing-slashes (semantic-settings/ee-embedding-service-base-url)) "/v1/embeddings")- (semantic-settings/ee-embedding-service-api-key)]+ (cond-> {:endpoint (str (trim-trailing-slashes (semantic-settings/ee-embedding-service-base-url))+ "/v1/embeddings")+ :api-key (semantic-settings/ee-embedding-service-api-key)}+ (setting/env-var-value :ee-embedding-service-base-url)+ (assoc :network-policy-floor :allow-private))(string? (not-empty (llm.settings/ai-service-base-url)))- [(str (trim-trailing-slashes (llm.settings/ai-service-base-url)) "/v1/embeddings")- nil]+ (cond-> {:endpoint (str (trim-trailing-slashes (llm.settings/ai-service-base-url)) "/v1/embeddings")}+ (setting/env-var-value :ai-service-base-url)+ (assoc :network-policy-floor :allow-private)):else(throw (ex-info "Embedding service and ai service base URLs are not configured"@@ -571,21 +589,22 @@"ai-service-base-url"]}))))(defmethod embedder-circuit-endpoint "ai-service" [_]- (first (embedding-service-resolve-config!)))+ (:endpoint (embedding-service-resolve-config!)))(defn- ai-service-get-embeddings-batch[{:keys [model-name vector-dimensions]} texts {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]- (let [[endpoint api-key] (embedding-service-resolve-config!)]+ (let [{:keys [endpoint api-key network-policy-floor]} (embedding-service-resolve-config!)](openai-compatible-get-embeddings-batch- {:provider "ai-service"- :endpoint endpoint- :api-key api-key- :model-name model-name- :vector-dimensions vector-dimensions- :texts texts- :snowplow? snowplow?- :record-tokens? record-tokens?- :type type})))+ {:provider "ai-service"+ :endpoint endpoint+ :api-key api-key+ :model-name model-name+ :vector-dimensions vector-dimensions+ :texts texts+ :snowplow? snowplow?+ :record-tokens? record-tokens?+ :type type+ :network-policy-floor network-policy-floor})));;;; OpenAI providerdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex 69303d9b3b8..ae2c6639344 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -11,6 +11,10 @@;; here, not there, so it's valid wherever the setter runs regardless of handler-namespace load order.(events/derive! :event/semantic-search-hnsw-enabled :metabase/event)+(defn- normalize-base-url+ [value]+ (some-> value str/trim not-empty))+(defsetting ee-embedding-provider(deferred-tru "The registered embedding provider to use"):encryption :when-encryption-key-set@@ -74,6 +78,15 @@:visibility :settings-manager:default nil:export? false+ ;; Normalize on read as well as write: environment values and rows persisted by older versions bypass the setter.+ :getter (fn []+ (normalize-base-url+ (setting/get-value-of-type :string :ee-embedding-service-base-url)))+ :setter (fn [new-value]+ (let [new-value (normalize-base-url new-value)]+ (when-let [problem (llm-settings/llm-url-problem new-value)]+ (throw (ex-info problem {:status-code 400})))+ (setting/set-value-of-type! :string :ee-embedding-service-base-url new-value))):doc false)(defsetting ee-embedding-service-api-keydiff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.cljindex 418aba5688d..f94799a2fb2 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/dlq_test.clj@@ -8,6 +8,7 @@[metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.llm.settings :as llm.settings][metabase.test :as mt][metabase.util.json :as json][next.jdbc :as jdbc]@@ -42,7 +43,12 @@(is (= :permanent (semantic.dlq/categorize-error (NullPointerException. "NPE")))))(testing "Default to transient"(is (= :transient (semantic.dlq/categorize-error (RuntimeException. "Unknown error"))))- (is (= :transient (semantic.dlq/categorize-error (Exception. "Generic exception"))))))+ (is (= :transient (semantic.dlq/categorize-error (Exception. "Generic exception")))))+ (testing "an embedding endpoint the network policy refuses is a configuration problem, not a passing one"+ (is (= :permanent (semantic.dlq/categorize-error+ (try (llm.settings/rethrow-if-llm-network-policy-error! (ex-info "blocked" {:ssrf true})+ "https://embed.example/v1")+ (catch Exception e e)))))))(deftest dlq-table-creation-test(testing "DLQ table creation and existence checks"diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.cljindex cc50496eeac..d1174aa897b 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj@@ -62,13 +62,16 @@(semantic.embedding/embedder-circuit-endpoint {:provider "openai"}))))))(deftest ^:synchronized request-failure-does-not-change-service-failure-history-test- (testing "request- and model-specific failures neither trip the circuit nor reset service-failure history"+ (testing "request-, model-, and policy-specific failures neither trip the circuit nor reset service-failure history"(let [breaker (dh.cb/circuit-breaker {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})](with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})](is (thrown? Exception (call-through boom)))(doseq [request-error [(ex-info "bad request" {:status 400})(ex-info "model not found" {:status 404})- (ex-info "wrong dimensions" {:cause :embedder/unexpected-dimensions})]]+ (ex-info "wrong dimensions" {:cause :embedder/unexpected-dimensions})+ (ex-info "network policy rejected the address" {:ssrf true})+ (ex-info "HTTP client wrapper" {}+ (ex-info "network policy rejected the address" {:ssrf true}))]](is (thrown? Exception (call-through #(throw request-error)))))(is (= :closed (circuit-state)))(is (thrown? Exception (call-through boom)))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.cljindex 301eef37d26..2cb7f5ffb6f 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj@@ -18,8 +18,10 @@[metabase.embeddings.provider :as embeddings.provider][metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features]+ [metabase.settings.core :as setting][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.http :as u.http][metabase.util.json :as json][toucan2.core :as t2])(:import@@ -378,6 +380,123 @@(is (= "Bearer embedding-api-key" (get-in @captured [:headers "Authorization"])))(is (nil? (get-in @captured [:headers "x-metabase-instance-token"]))))))))))+(deftest embedding-endpoints-honor-llm-allowed-networks-test+ ;; IP literals throughout: the resolver goes through real DNS. `capture` stands in for clj-http far enough to+ ;; run the request's `:dns-resolver` on its host, which is where the policy is enforced.+ (let [mock-response {:data [{:object "embedding"+ :embedding (encode-floats-to-base64 [1.0 2.0 3.0 4.0])+ :index 0}]+ :model "test-model"+ :usage {:prompt_tokens 1 :total_tokens 1}}+ model {:model-name "test-model" :vector-dimensions 4}+ captured (atom nil)+ capture (fn [url opts]+ (some-> ^org.apache.http.conn.DnsResolver (:dns-resolver opts)+ (.resolve (u.http/->hostname url)))+ (reset! captured (assoc opts :url url))+ {:status 200+ :headers {"Content-Type" "application/json"}+ :body (json/encode mock-response)})+ embed (fn [provider]+ (embedding/get-embedding (assoc model :provider provider) "text" {:record-tokens? false}))+ rejected (fn [provider]+ (try (embed provider)+ nil+ (catch clojure.lang.ExceptionInfo e (ex-data e))))+ loopback (constantly "http://127.0.0.1:9")]+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (testing "the OpenAI base URL is admin input and is refused on an internal network"+ (mt/with-dynamic-fn-redefs [llm.settings/llm-openai-api-key (constantly "sk-test")+ llm.settings/llm-openai-api-base-url loopback+ http/post capture]+ (is (=? {:status-code 400 :status 400 :error-code :llm-host-not-allowed :llm-host "127.0.0.1"}+ (rejected "openai")))))+ (testing "so is the embedding service URL"+ (mt/with-dynamic-fn-redefs [semantic.settings/ee-embedding-service-base-url loopback+ semantic.settings/ee-embedding-service-api-key (constantly "key")+ http/post capture]+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed} (rejected "ai-service")))))+ (testing "a permitted OpenAI base URL goes out with the policy-enforcing DNS resolver on the connection"+ (mt/with-dynamic-fn-redefs [llm.settings/llm-openai-api-key (constantly "sk-test")+ llm.settings/llm-openai-api-base-url (constantly "https://8.8.8.8")+ http/post capture]+ (embed "openai")+ (is (= "https://8.8.8.8/v1/embeddings" (:url @captured)))+ (is (= :none (:redirect-strategy @captured)))+ (is (instance? org.apache.http.conn.DnsResolver (:dns-resolver @captured)))))+ (testing "a connection-time DNS policy rejection has the same 400 shape as the upfront check"+ (mt/with-dynamic-fn-redefs [llm.settings/llm-openai-api-key (constantly "sk-test")+ llm.settings/llm-openai-api-base-url (constantly "https://8.8.8.8")+ http/post (fn [& _]+ (throw (ex-info "blocked"+ {:ssrf true})))]+ (is (=? {:status-code 400 :api-error true :error-code :llm-host-not-allowed}+ (rejected "openai")))))+ (testing "a stored AI service URL gets the default policy: private is refused"+ (mt/with-dynamic-fn-redefs [semantic.settings/ee-embedding-service-base-url (constantly nil)+ llm.settings/ai-service-base-url (constantly "http://10.0.0.1:9")+ premium-features/premium-embedding-token (constantly "mock-token")+ http/post capture]+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed}+ (rejected "ai-service")))))+ (testing "an AI service URL the environment names is deployment configuration: its floor admits private"+ (mt/with-premium-features #{:metabot-v3}+ (mt/with-temp-env-var-value! [mb-ai-service-base-url "http://10.0.0.1:9"]+ (mt/with-dynamic-fn-redefs [semantic.settings/ee-embedding-service-base-url (constantly nil)+ premium-features/premium-embedding-token (constantly "mock-token")+ http/post capture]+ (embed "ai-service")+ (is (= "http://10.0.0.1:9/v1/embeddings" (:url @captured)))+ (is (instance? org.apache.http.conn.DnsResolver (:dns-resolver @captured)))))))+ (testing "the environment-supplied floor still refuses loopback"+ (mt/with-premium-features #{:metabot-v3}+ (mt/with-temp-env-var-value! [mb-ai-service-base-url "http://127.0.0.1:9"]+ (mt/with-dynamic-fn-redefs [semantic.settings/ee-embedding-service-base-url (constantly nil)+ premium-features/premium-embedding-token (constantly "mock-token")+ http/post capture]+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed}+ (rejected "ai-service")))))))+ (testing "an embedding service URL the environment names is deployment configuration: private is fine"+ (mt/with-temp-env-var-value! [mb-ee-embedding-service-base-url "http://10.0.0.1:9"]+ (mt/with-dynamic-fn-redefs [semantic.settings/ee-embedding-service-api-key (constantly "key")+ http/post capture]+ (embed "ai-service")+ (is (= "http://10.0.0.1:9/v1/embeddings" (:url @captured)))+ (is (instance? org.apache.http.conn.DnsResolver (:dns-resolver @captured))))))+ (testing "the embedding service URL is checked on write as well"+ (mt/with-premium-features #{}+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"not allowed to connect"+ (semantic.settings/ee-embedding-service-base-url! "http://127.0.0.1:9"))))))+ (testing "under :allow-all an internal OpenAI base URL goes out on clj-http's default resolver"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-dynamic-fn-redefs [llm.settings/llm-openai-api-key (constantly "sk-test")+ llm.settings/llm-openai-api-base-url loopback+ http/post capture]+ (embed "openai")+ (is (= "http://127.0.0.1:9/v1/embeddings" (:url @captured)))+ (is (not (contains? @captured :dns-resolver))))))))++(deftest embedding-service-base-url-normalizes-whitespace-test+ (mt/with-temporary-setting-values [ee-embedding-service-base-url nil]+ (testing "new writes"+ (semantic.settings/ee-embedding-service-base-url! " \t ")+ (is (nil? (semantic.settings/ee-embedding-service-base-url))+ "whitespace clears the setting instead of leaving it looking configured")+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (semantic.settings/ee-embedding-service-base-url! " https://embed.example.com/v1 ")+ (is (= "https://embed.example.com/v1" (semantic.settings/ee-embedding-service-base-url)))))+ (testing "a whitespace-only row written by an older version"+ (setting/set-value-of-type! :string :ee-embedding-service-base-url " \t ")+ (mt/with-dynamic-fn-redefs [llm.settings/ai-service-base-url (constantly "https://ai.example.com")]+ (is (= "https://ai.example.com/v1/embeddings"+ (embedding/embedder-circuit-endpoint {:provider "ai-service"})))))+ (testing "a whitespace-only environment value"+ (mt/with-temp-env-var-value! [mb-ee-embedding-service-base-url " \t "]+ (mt/with-dynamic-fn-redefs [llm.settings/ai-service-base-url (constantly "https://ai.example.com")]+ (is (= "https://ai.example.com/v1/embeddings"+ (embedding/embedder-circuit-endpoint {:provider "ai-service"}))))))))+(deftest test-embedding-service-snowplow-tracking(testing "ai-service fires a Snowplow token_usage event on each batch call"(mt/with-temporary-setting-values [ee-embedding-service-base-url "http://mock-embedding-service"@@ -443,7 +562,7 @@(with-redefs [semantic.settings/ee-embedding-provider (constantly provider)semantic.settings/ee-embedding-model (constantly "mock-model")semantic.settings/openai-api-key (constantly "xyz")- semantic.settings/openai-api-base-url (constantly "xyz")+ semantic.settings/openai-api-base-url (constantly "https://mock-openai")semantic.settings/ee-embedding-service-base-url (constantly "http://mock-embedding-service")semantic.settings/ee-embedding-service-api-key (constantly "mock-key")http/post (fn post-mock [_url {:keys [body]}]diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.cljindex 5017259be6e..d7141d3ae72 100644--- a/src/metabase/driver/settings.clj+++ b/src/metabase/driver/settings.clj@@ -13,7 +13,7 @@(defsetting warehouse-allowed-networks(deferred-tru (str "Controls which networks Metabase may connect to for warehouse connections.\n""Options:\n"- "- external-only (only globally routable public addresses)\n"+ "- external-only (only globally reachable public addresses)\n""- allow-private (external + private networks but NOT loopback or link-local)\n""- allow-all (no restrictions).\n""Defaults to external-only on Metabase Cloud and allow-all when self-hosted.\n"diff --git a/src/metabase/llm/anthropic.clj b/src/metabase/llm/anthropic.cljindex 6c1eb7f5be1..5a1fb7915e3 100644--- a/src/metabase/llm/anthropic.clj+++ b/src/metabase/llm/anthropic.clj@@ -96,24 +96,27 @@:messages messages}start-time (u/start-timer)url (str (llm.settings/llm-anthropic-api-base-url) "/v1/messages")]- ;; Outside the try so the e2e guard fails loudly instead of being routed- ;; through `handle-api-error` (mirrors `metabase.metabot.self.core/request`).+ ;; Outside the try so the e2e guard and a malformed URL fail loudly instead of being+ ;; routed through `handle-api-error` (mirrors `metabase.metabot.self.core/request`).(llm.settings/assert-llm-host-allowed! url)- (try- (let [response (http/post url- {:headers (build-request-headers (get-api-key-or-throw))- :body (json/encode (build-request-body request))- :as :json- :content-type :json- :socket-timeout (llm.settings/llm-request-timeout-ms)- :connection-timeout (llm.settings/llm-connection-timeout-ms)})- duration-ms (u/since-ms start-time)- body (:body response)- usage (:usage body)]- {:result (extract-tool-input body)- :duration-ms duration-ms- :usage {:model model- :prompt (:input_tokens usage)- :completion (:output_tokens usage)}})- (catch Exception e- (handle-api-error e)))))+ (let [policy-opts (llm.settings/llm-request-opts url)]+ (try+ (let [response (http/post url+ (merge {:headers (build-request-headers (get-api-key-or-throw))+ :body (json/encode (build-request-body request))+ :as :json+ :content-type :json+ :socket-timeout (llm.settings/llm-request-timeout-ms)+ :connection-timeout (llm.settings/llm-connection-timeout-ms)}+ policy-opts))+ duration-ms (u/since-ms start-time)+ body (:body response)+ usage (:usage body)]+ {:result (extract-tool-input body)+ :duration-ms duration-ms+ :usage {:model model+ :prompt (:input_tokens usage)+ :completion (:output_tokens usage)}})+ (catch Exception e+ (llm.settings/rethrow-if-llm-network-policy-error! e url)+ (handle-api-error e))))))diff --git a/src/metabase/llm/provider.clj b/src/metabase/llm/provider.cljindex 5b679d69a25..d7e25567045 100644--- a/src/metabase/llm/provider.clj+++ b/src/metabase/llm/provider.clj@@ -57,6 +57,7 @@:docs-url "https://console.anthropic.com/settings/keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -74,6 +75,7 @@:docs-url "https://platform.openai.com/api-keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -91,6 +93,7 @@:docs-url "https://openrouter.ai/keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -108,6 +111,7 @@:docs-url "https://console.mistral.ai/api-keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -125,6 +129,7 @@:docs-url "https://z.ai/manage-apikey/apikey-list"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -141,6 +146,7 @@:docs-url "https://platform.kimi.ai/console/api-keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -160,6 +166,7 @@:docs-url "https://platform.deepseek.com/api_keys"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -230,6 +237,7 @@:help (deferred-tru "A short-lived token, e.g. the output of gcloud auth print-access-token. Useful for testing.")}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -250,6 +258,7 @@:docs-url "https://ai.azure.com"}{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:required? true@@ -309,11 +318,14 @@:default-model nil:fields [{:key :base-url:normalize strip-trailing-slashes+ :validate llm.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:required? true- :placeholder "http://vllm.internal:8000/v1"- :help (deferred-tru "Your server''s OpenAI-compatible API. It should end in /v1.")}+ :placeholder "https://vllm.example.com/v1"+ :help (deferred-tru (str "Your server''s OpenAI-compatible API. It should end in /v1. "+ "Metabase must be able to reach it: self-hosted, a server on your "+ "private network or on this machine needs MB_LLM_ALLOWED_NETWORKS."))}{:key :api-key:label (deferred-tru "API key"):type :password@@ -431,8 +443,14 @@(when (every? some? parts)(str/join "/" parts)))))+(defn- validate-field-value!+ "Run one field's `:validate` hook against the value `config` supplies for it, if it has both."+ [{:keys [key validate]} config]+ (when-let [problem (and validate (some-> (u/trimmed-string (get config key)) validate))]+ (throw (ex-info (str problem) {:status-code 400 :field key}))))+(defn- validate-field!- [type-name {:keys [key label required? prefix default options validate]} config]+ [type-name {:keys [key label required? prefix default options] :as field} config](let [value (u/trimmed-string (get config key))](when (and required? (not value) (not default))(throw (ex-info (tru "{0} is required for {1}." (str label) type-name)@@ -443,8 +461,7 @@(when (and value (seq options) (not-any? #(= value (:value %)) options))(throw (ex-info (tru "Invalid {0} for {1}." (str label) type-name){:status-code 400 :field key})))- (when-let [problem (and value validate (validate value))]- (throw (ex-info (str problem) {:status-code 400 :field key})))))+ (validate-field-value! field config)))(defn- validate-config-field!"Run [[validate-field!]]'s checks for the single field `field-key` of `type-name` against `config`."@@ -732,6 +749,30 @@(when-let [{:keys [type config]} (connection conn-key)](config-complete? type config))))+(defn validate-changed-connections!+ "Run the per-field `:validate` hooks over the fields `conns` changes. This is the+ [[metabase.llm.settings/llm-providers]] setter, so a base URL the network policy refuses cannot be saved by writing+ the connection list straight through `PUT /api/setting/llm-providers` or `config.yml` rather than through the+ connection API.++ Only the `:validate` hooks: required fields, prefixes and options are the connection API's business, and demanding+ them of every write here would break `config.yml` provisioning and the single-provider settings, which+ legitimately fill a connection in one field at a time.++ Only the fields that changed, too, so that a base URL saved before this check existed -- or before the policy was+ tightened around it -- does not make the connection holding it unwritable. Rotating its API key has to keep+ working.++ A connection of an unknown type has no fields to check, the same as everywhere else a hand-written list is read."+ [conns]+ (let [stored (into {} (map (juxt (juxt :key :type) :config)) (stored-connections))]+ (doseq [conn (filter map? conns)+ :let [config (:config conn)+ previous (get stored ((juxt :key :type) conn))]+ {field-key :key :as field} (:fields (provider-type (:type conn)))+ :when (not= (get config field-key) (get previous field-key))]+ (validate-field-value! field config))))+(defn set-connections!"Persist `conns` as the stored connection list, dropping the derived annotation keys."[conns]diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.cljindex cbc0d9256cb..45eb5279d46 100644--- a/src/metabase/llm/settings.clj+++ b/src/metabase/llm/settings.clj@@ -4,9 +4,11 @@[clojure.string :as str][metabase.config.core :as config][metabase.premium-features.core :as premium-features]- [metabase.settings.core :refer [defsetting]]+ [metabase.settings.core :as setting :refer [defsetting]][metabase.util :as u]- [metabase.util.i18n :refer [deferred-tru tru]])+ [metabase.util.http :as u.http]+ [metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.log :as log])(:import(java.net MalformedURLException URL)(software.amazon.awssdk.regions Region)))@@ -41,6 +43,174 @@{:status-code 400:llm-url url}))))))+(def ^:private network-policies+ "The `llm-allowed-networks` policies, loosest last."+ [:external-only :allow-private :allow-all])++(def ^:private network-policy-rank+ (zipmap network-policies (range)))++(defonce ^:private warned-network-policy-values+ (atom #{}))++(defsetting llm-allowed-networks+ (deferred-tru (str "Controls which networks Metabase may connect to for LLM provider base URLs. "+ "Set through the environment only; on Metabase Cloud the default applies.\n"+ "Options:\n"+ "- external-only (default; only globally reachable public addresses)\n"+ "- allow-private (external + private networks but NOT loopback or link-local)\n"+ "- allow-all (no restrictions).\n"+ "The Metabase AI service and LLM proxy are deployment configuration and may always use "+ "private addresses."))+ :type :keyword+ ;; Environment only. A settings manager is who this policy defends against, and on Cloud a customer admin+ ;; loosening it would be reaching for our own infrastructure, so nobody sets it through the API, and a value+ ;; that reached the app DB some other way is ignored rather than trusted.+ :visibility :internal+ :setter :none+ :default :external-only+ :export? false+ :doc (str "Set this when a self-hosted vLLM server is on your private network (allow-private) or on this "+ "machine (allow-all). There is no admin UI for it, and a value stored in the application "+ "database is ignored. With a JVM-wide HTTP(S) proxy, Metabase checks destination addresses "+ "available through local DNS; the deployment proxy must enforce destination restrictions on "+ "its outbound connections. Proxy-only DNS is supported. Metabase enforces destination addresses "+ "at connection time for direct requests.")+ :getter (fn []+ (let [value (some-> (setting/env-var-value :llm-allowed-networks) keyword)]+ (cond+ (nil? value) :external-only+ (contains? network-policy-rank value) value+ ;; fail closed on a typo, and say so once rather than on every request+ :else+ (do (when-not (contains? @warned-network-policy-values value)+ (swap! warned-network-policy-values conj value)+ (log/warnf "Ignoring MB_LLM_ALLOWED_NETWORKS=%s: expected one of %s; using external-only"+ (name value) (str/join ", " (map name network-policies))))+ :external-only)))))++(defn network-policy+ "The network policy for an LLM request.+ `floor`, for a deployment-controlled endpoint such as the AI service, can only loosen [[llm-allowed-networks]]:+ the looser of the two applies."+ ([]+ (llm-allowed-networks))+ ([floor]+ (let [configured (llm-allowed-networks)]+ (if (and floor (> (network-policy-rank floor) (network-policy-rank configured)))+ floor+ configured))))++(defn- host-not-allowed-message+ "Why a base URL on `host` is refused under `policy`, and what to do about it.+ `policy` is the one that actually refused, which a deployment-controlled endpoint's floor may have loosened past+ [[llm-allowed-networks]]: naming a value that is already in force would be advice that changes nothing.+ On Cloud the policy is not the customer's to change."+ [policy host]+ (cond+ (premium-features/is-hosted?)+ (tru "The base URL host {0} is not permitted by Metabase Cloud''s LLM network policy. Use an LLM provider on the public internet." host)++ (= :allow-private policy)+ (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-all for a server on this machine." host)++ :else+ (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-private for a server on your private network, or allow-all for one on this machine." host)))++(defn- url-not-allowed-ex+ "The 400 every policy refusal is thrown as, at set time and at connection time alike.+ `:status` sits beside `:status-code` because the semantic-search dead-letter queue files errors by `:status`, and a+ refused endpoint is a permanent failure, not one to retry on the fast schedule."+ ([message host]+ (url-not-allowed-ex message host nil))+ ([message host cause]+ (ex-info message+ {:status-code 400+ :status 400+ :api-error true+ :error-code :llm-host-not-allowed+ :llm-host host}+ cause)))++(defn llm-url-syntax-problem+ "Why `url` cannot be an LLM provider base URL under any policy, or nil when it can: it must be an `http` or+ `https` URL that names a host and carries no username or password.+ Nothing here resolves the host, so it is cheap enough to run on every request.+ A blank `url` is not a problem here: the not-configured handling covers it."+ [url]+ (when-not (str/blank? url)+ (let [^URL parsed (try+ (URL. ^String url)+ (catch MalformedURLException _ nil))]+ (cond+ (not (and parsed (#{"http" "https"} (.getProtocol parsed)) (not-empty (.getHost parsed))))+ (tru "Invalid base URL: it must start with http:// or https://.")++ ;; it would otherwise ride along into error messages and ex-data+ (some? (.getUserInfo parsed))+ (tru "Invalid base URL: it must not contain a username or password.")))))++(defn llm-url-problem+ "Why `url` may not be used as an LLM provider base URL, or nil when it may: [[llm-url-syntax-problem]], and every+ address the host resolves to must be permitted by the network policy. The one-argument form uses+ [[llm-allowed-networks]].+ Used on write and before proxied requests. An unresolved host is permitted to allow proxy-only DNS.+ Direct requests use the policy resolver from [[llm-request-opts]] to enforce the policy at connection time."+ ([url]+ (llm-url-problem (llm-allowed-networks) url))+ ([network-policy url]+ (or (llm-url-syntax-problem url)+ (when-not (str/blank? url)+ (let [host (.getHost (URL. ^String url))]+ (when-not (u.http/host-allowed-for-network-policy? network-policy host)+ (host-not-allowed-message network-policy host)))))))++(defn llm-request-opts+ "clj-http options that put the network policy on a request to `url`: redirects are never followed, and a+ `:dns-resolver` refuses any address the policy does not permit (omitted under `:allow-all`). Throws the same 400 as+ a set-time refusal when `url` fails [[llm-url-syntax-problem]]. `floor` is for a deployment-controlled endpoint,+ see [[network-policy]]. Pair with [[rethrow-if-llm-network-policy-error!]] around the request.++ Behind a JVM-wide proxy a `:dns-resolver` enforces nothing, so the check moves to [[llm-url-problem]] here; see+ [[metabase.util.http/jvm-proxied-url?]]."+ ([url]+ (llm-request-opts nil url))+ ([floor url]+ (when-let [problem (llm-url-syntax-problem url)]+ (throw (url-not-allowed-ex problem (u.http/->hostname url))))+ (let [policy (network-policy floor)]+ (if (u.http/jvm-proxied-url? url)+ ;; An operator who configured a JVM proxy put it between Metabase and everything, deliberately, so it is+ ;; trusted rather than judged by the policy -- which is also what makes a private egress proxy usable under+ ;; `:external-only`. The proxy resolves the target on its own, so the target is checked here instead. That+ ;; leaves proxy-only DNS, differing DNS answers, and rebinding to the proxy's own destination policy;+ ;; Metabase cannot enforce the addresses the proxy connects to.+ (do (when-let [problem (llm-url-problem policy url)]+ (throw (url-not-allowed-ex problem (u.http/->hostname url))))+ {:redirect-strategy :none})+ ;; nil under :allow-all, which leaves clj-http on its default resolver. Redirects stay disabled under every+ ;; policy: the resolver would stop an internal target, but credentials could otherwise follow a redirect to a+ ;; public host.+ (u/assoc-dissoc {:redirect-strategy :none}+ :dns-resolver (u.http/network-policy-dns-resolver policy))))))++(defn llm-network-policy-error?+ "Whether `e` or one of its causes is a connection-time refusal from the policy DNS resolver."+ [e]+ (boolean (:ssrf (u/all-ex-data e))))++(defn rethrow-if-llm-network-policy-error!+ "Translate a connection-time refusal from the policy DNS resolver in `e` to the 400 a set-time refusal gets.+ Returns nil when `e` is unrelated. Logs the refusal: it is the only trace a host that rebinds leaves.+ The resolver records the policy it refused under, which is what the message and the log line report."+ [e url]+ (when (llm-network-policy-error? e)+ (let [host (u.http/->hostname url)+ policy (or (:policy (u/all-ex-data e)) (llm-allowed-networks))]+ (log/warnf "Refused an LLM request to %s: it resolves to an address the %s network policy does not permit"+ host (name policy))+ (throw (url-not-allowed-ex (host-not-allowed-message policy host) host e)))))+;; TODO (Chris 2026-08-17) -- BOT-2005: generate-sql and semantic search read these settings directly, so;; deleting the connection they key off turns those features off. They should name a connection instead.(defn- connection-field-getter@@ -452,6 +622,12 @@:visibility :settings-manager:export? false:audit :no-value+ ;; The connection API validates what it saves, and so do the single-provider settings, but the list is also+ ;; writable as itself -- through `PUT /api/setting/llm-providers` and through `config.yml` -- so the field+ ;; validators run here too, on the way in.+ :setter (fn [new-value]+ ((requiring-resolve 'metabase.llm.provider/validate-changed-connections!) new-value)+ (setting/set-value-of-type! :json :llm-providers new-value)):doc "Connections are normally managed from the admin AI settings page. Setting this environment variable puts the whole list under environment control and makes it read-only in the UI.Configuring a provider through the single-provider variables (`MB_LLM_ANTHROPIC_API_KEY` and friends) is equally supported, and is the simpler option when you only need one connection per provider and would rather not hand-write JSON. Each such provider becomes a read-only connection whose key is the provider type, resolved from the environment on every read, so editing one of those variables is picked up on the next restart. A provider configured this way takes precedence over a stored connection with the same key.")diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 6e601788420..00818000b70 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -11,6 +11,7 @@[metabase.llm.settings :as llm][metabase.metabot.schema.v2 :as schema.v2][metabase.premium-features.core :as premium-features]+ [metabase.settings.core :as setting][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]@@ -1039,12 +1040,19 @@(defn resolve-auth"Pick the right auth map for an LLM request.- - When `ai-proxy?` is true, uses the Metabase Cloud proxy (errors if unconfigured).- - Otherwise uses the provider's BYOK `auth`."+ - When `ai-proxy?` is true, uses the Metabase Cloud proxy (errors if unconfigured). When the environment supplies+ the proxy URL it carries a `:network-policy-floor` of `:allow-private`, so private cluster addresses remain+ reachable under the default policy; see [[metabase.llm.settings/network-policy]].+ - Otherwise uses the provider's BYOK `auth`."[provider-slug llm-type auth ai-proxy?](let [proxy-auth (when-let [base (llm/llm-proxy-base-url)]- {:url (str (str/replace base #"/+$" "") "/" provider-slug)- :headers {"x-metabase-instance-token" (premium-features/premium-embedding-token)}})]+ (cond-> {:url (str (str/replace base #"/+$" "") "/" provider-slug)+ :headers {"x-metabase-instance-token"+ (premium-features/premium-embedding-token)}}+ ;; only an environment-supplied URL is deployment-controlled: a superuser can write the+ ;; stored setting through the generic settings API, which must not widen the policy+ (setting/env-var-value :llm-proxy-base-url)+ (assoc :network-policy-floor :allow-private)))](if ai-proxy?(or proxy-auth(throw (ex-info (tru "AI proxy is not configured")@@ -1057,14 +1065,27 @@"Perform an LLM HTTP request with the given auth (a map of `:url` and `:headers`).Forces a connection + socket timeout on every request so a hung upstream cannever block the caller forever. The timeouts default to the operator-tunable- `llm/llm-connection-timeout-ms` and `llm/llm-request-timeout-ms` settings (read+ [[metabase.llm.settings/llm-connection-timeout-ms]] and+ [[metabase.llm.settings/llm-request-timeout-ms]] settings (readat call time), the same knobs `metabase.llm.anthropic` uses. Callers canoverride either timeout per request by passing `:connection-timeout` /- `:socket-timeout` in `req`."- [{:keys [url headers]} req]+ `:socket-timeout` in `req`.++ The connection resolves DNS through a resolver that enforces+ [[metabase.llm.settings/llm-allowed-networks]] on the addresses it actually+ opens; see [[metabase.llm.settings/llm-request-opts]]. Auth returned by+ [[resolve-auth]] may supply `:network-policy-floor` for a+ deployment-controlled service."+ [{:keys [url headers network-policy-floor]} req](llm/assert-llm-host-allowed! url)- (http/request (-> {:connection-timeout (llm/llm-connection-timeout-ms)- :socket-timeout (llm/llm-request-timeout-ms)}- (merge req)- (update :url #(str url %))- (update :headers merge headers))))+ (let [policy-opts (llm/llm-request-opts network-policy-floor url)]+ (try+ (http/request (-> {:connection-timeout (llm/llm-connection-timeout-ms)+ :socket-timeout (llm/llm-request-timeout-ms)}+ (merge req)+ (update :url #(str url %))+ (update :headers merge headers)+ (merge policy-opts)))+ (catch Exception e+ (llm/rethrow-if-llm-network-policy-error! e url)+ (throw e)))))diff --git a/src/metabase/metabot/self/google.clj b/src/metabase/metabot/self/google.cljindex b9d8be41a8f..252a59ce0bc 100644--- a/src/metabase/metabot/self/google.clj+++ b/src/metabase/metabot/self/google.clj@@ -58,6 +58,25 @@There is also a `cloud-platform.read-only` scope, but inference calls made with the read-only scope are rejected.""https://www.googleapis.com/auth/cloud-platform")+(def ^:private google-token-uris+ "The complete `token_uri` values a service account key may carry: Google's current OAuth token endpoint and the+ legacy one older keys still name. Whole URIs rather than hosts, because the transport posts whatever the key+ says: a port, a query string, or another path on the same origin is not one of these endpoints."+ #{"https://oauth2.googleapis.com/token"+ "https://accounts.google.com/o/oauth2/token"})++(defn- check-token-uri!+ "Refuse a key whose `token_uri` is not one of Google's.+ The credential library posts to that URL to mint an access token, before any request our own network policy+ guards, so an admin-supplied key could otherwise point it at an internal host."+ [^ServiceAccountCredentials creds]+ (let [uri (.getTokenServerUri creds)]+ (when-not (contains? google-token-uris (u/lower-case-en (str uri)))+ (throw (ex-info (tru "Invalid Google service account key: token_uri must be a Google OAuth endpoint.")+ {:api-error true+ :status-code 400+ :error-code :invalid-service-account-key})))))+(defn- parse-service-account-credentials"Parses a service account key JSON string into scoped `ServiceAccountCredentials`."^ServiceAccountCredentials [^String sa-key]@@ -76,6 +95,7 @@{:api-error true:status-code 400:error-code :not-a-service-account-key})))+ (check-token-uri! creds)(.createScoped ^ServiceAccountCredentials creds (Collections/singletonList cloud-platform-scope))))(def ^:private cached-service-account-credentialsdiff --git a/src/metabase/premium_features/api.clj b/src/metabase/premium_features/api.cljindex f78a6549a81..bade11e8062 100644--- a/src/metabase/premium_features/api.clj+++ b/src/metabase/premium_features/api.clj@@ -7,6 +7,7 @@[metabase.llm.settings :as llm.settings][metabase.premium-features.core :as premium-features][metabase.premium-features.token-check :as token-check]+ [metabase.settings.core :as setting][metabase.util.log :as log][ring.util.response :as response])(:import@@ -25,14 +26,24 @@(api/check-404 (premium-features/token-status)))(defn- invalidate-llm-proxy-token-cache!- "Invalidation of the AI service's cached token status for the current instance token."+ "Invalidation of the AI service's cached token status for the current instance token.++ The URL carries the instance token in its path, and `ai-service-base-url` is a stored setting a superuser can+ write through the generic settings API, so this request gets the same network policy as any other LLM request:+ see [[metabase.llm.settings/llm-request-opts]]. Only a base URL the environment supplies is deployment+ configuration, and only that one is granted the `:allow-private` floor a service inside the cluster needs.++ Best effort: a refused or failed invalidation is logged, not raised. Token refresh has already happened by the+ time it runs, and the AI service's cache expires on its own."[](when-let [service-base-url (llm.settings/ai-service-base-url)](when-let [^String token (premium-features/premium-embedding-token)](try(let [encoded-token (URLEncoder/encode ^String token "UTF-8")url (str (str/replace service-base-url #"/+$" "") "/v1/invalidate-token-cache/" encoded-token)- response (http/post url {:throw-exceptions false})]+ floor (when (setting/env-var-value :ai-service-base-url) :allow-private)+ response (http/post url (merge {:throw-exceptions false}+ (llm.settings/llm-request-opts floor url)))](when-not (<= 200 (:status response) 299)(log/warnf "LLM proxy token cache invalidation failed with status %s" (:status response))))(catch Exception ediff --git a/src/metabase/util/http.clj b/src/metabase/util/http.cljindex 3badfb82bee..cf40a87f2fd 100644--- a/src/metabase/util/http.clj+++ b/src/metabase/util/http.clj@@ -7,10 +7,13 @@(:import(com.google.common.net InetAddresses)(java.io ByteArrayOutputStream InputStream)- (java.net Inet6Address InetAddress URI URL)- (java.util Locale)+ (java.net Inet6Address InetAddress ProxySelector URI URL)+ (java.util Arrays Locale)+ (org.apache.http HttpHost)(org.apache.http.conn DnsResolver)- (org.apache.http.impl.conn SystemDefaultDnsResolver)))+ (org.apache.http.impl.conn SystemDefaultDnsResolver SystemDefaultRoutePlanner)+ (org.apache.http.message BasicHttpRequest)+ (org.apache.http.protocol BasicHttpContext)))(set! *warn-on-reflection* true)@@ -37,7 +40,8 @@;; - Validate every *resolved* IP is a public unicast address via a custom DnsResolver -- this;; runs inside the connection the client actually opens, closing the DNS-rebinding TOCTOU gap.;; It rejects loopback, link-local (incl. cloud metadata 169.254.169.254), site-local (RFC1918),-;; any-local, multicast, IPv6 ULA (fc00::/7), and IPv4 CGNAT (100.64/10).+;; any-local, multicast, IPv6 ULA (fc00::/7), IPv4 CGNAT (100.64/10), non-global IANA+;; special-purpose ranges, and IPv6 space IANA has not allocated.;; - No redirects (a 3xx would be a bypass vector; here it just fails).;; - No cookies/credentials (a fresh clj-http GET carries no Metabase session).;; - Cap the download bytes and (optionally) restrict to an allowlist of content-types.@@ -50,31 +54,126 @@(def ^:private blocked-fetch-hosts #{"localhost" "metadata" "metadata.google.internal"})(def ^:private blocked-fetch-host-suffixes [".localhost" ".local" ".internal" ".lan" ".home.arpa"])+(defn- address-prefix+ [address prefix-length]+ (let [bytes (.getAddress ^InetAddress (InetAddresses/forString address))]+ (assert (<= 0 prefix-length (* 8 (alength bytes))))+ [bytes prefix-length]))++(defn- address-in-prefix?+ [^InetAddress addr [network prefix-length]]+ (let [address-bytes (.getAddress addr)+ ^bytes network network+ whole-byte-count (quot prefix-length 8)+ remaining-bits (mod prefix-length 8)]+ (and (= (alength address-bytes) (alength network))+ (loop [i 0]+ (or (= i whole-byte-count)+ (and (= (aget address-bytes i) (aget network i))+ (recur (inc i)))))+ (or (zero? remaining-bits)+ (let [mask (bit-and 0xff (bit-shift-left 0xff (- 8 remaining-bits)))]+ (= (bit-and (aget address-bytes whole-byte-count) mask)+ (bit-and (aget network whole-byte-count) mask)))))))++(def ^:private globally-reachable-special-prefixes+ ;; More-specific exceptions inside the broad non-global prefixes below. Keep this aligned with the IANA IPv4 and+ ;; IPv6 Special-Purpose Address Registries' "Globally Reachable" column.+ ;; https://www.iana.org/assignments/iana-ipv4-special-registry+ ;; https://www.iana.org/assignments/iana-ipv6-special-registry+ [(address-prefix "192.0.0.9" 32) ; PCP anycast+ (address-prefix "192.0.0.10" 32) ; TURN anycast+ (address-prefix "2001:1::1" 128) ; PCP anycast+ (address-prefix "2001:1::2" 128) ; TURN anycast+ (address-prefix "2001:1::3" 128) ; DNS-SD service registration anycast+ (address-prefix "2001:3::" 32) ; AMT+ (address-prefix "2001:4:112::" 48) ; AS112-v6+ (address-prefix "2001:20::" 28) ; ORCHIDv2+ (address-prefix "2001:30::" 28)]) ; Drone Remote ID protocol entity tags++(def ^:private non-global-special-prefixes+ ;; Additional IANA special-purpose blocks that are not globally reachable and are not already handled by+ ;; `InetAddress` or the checks in [[public-address?]]. Entries whose registry value is N/A or blank are also refused:+ ;; they do not carry the external-reachability guarantee required by `:external-only`. IPv6 blocks outside 2000::/3+ ;; (discard-only, local-use translation, segment-routing SIDs, ...) need no entry: everything outside the+ ;; global-unicast space is refused wholesale unless listed above.+ ;; https://www.iana.org/assignments/iana-ipv4-special-registry+ ;; https://www.iana.org/assignments/iana-ipv6-special-registry+ [(address-prefix "192.0.0.0" 24) ; IETF protocol assignments+ (address-prefix "192.0.2.0" 24) ; TEST-NET-1+ (address-prefix "192.88.99.0" 24) ; deprecated 6to4 relay anycast+ (address-prefix "198.18.0.0" 15) ; benchmarking+ (address-prefix "198.51.100.0" 24) ; TEST-NET-2+ (address-prefix "203.0.113.0" 24) ; TEST-NET-3+ (address-prefix "2001::" 23) ; IETF protocol assignments+ (address-prefix "2001:db8::" 32) ; documentation+ (address-prefix "2002::" 16)]) ; 6to4++(def ^:private reserved-global-unicast-prefixes+ ;; 2000::/3 is the global-unicast space, but everything in it from 2d00:: up is unallocated: the fifteen blocks+ ;; the registry marks RESERVED -- which include the 3fff::/20 documentation block -- and the space above them it+ ;; does not list at all. No ISP routes any of it, so a name resolving there is either a mistake or an internal+ ;; network squatting on reserved space. Shrink this when IANA allocates from the top of the /3.+ ;; The smaller unallocated holes below 2d00:: are left alone: they sit between live RIR allocations, which is+ ;; where the next ones are handed out from, and refusing them would age badly.+ ;; https://www.iana.org/assignments/ipv6-unicast-address-assignments+ [(address-prefix "2d00::" 8)+ (address-prefix "2e00::" 7)+ (address-prefix "3000::" 4)])++(def ^:private non-global-prefixes+ "Every prefix `:external-only` refuses, unless [[globally-reachable-special-prefixes]] carves it back out."+ (into non-global-special-prefixes reserved-global-unicast-prefixes))++(def ^:private nat64-well-known-prefix+ ;; 64:ff9b::/96, whose low 32 bits are the IPv4 address a NAT64 gateway translates to (RFC 6052).+ (address-prefix "64:ff9b::" 96))++(defn- translated-address+ "The address a connection to `addr` actually reaches: the IPv4 embedded in the NAT64 well-known prefix, or `addr`+ itself. RFC 6052 says the well-known prefix may not carry a non-global IPv4, but nothing on this side of the+ gateway enforces that, and `64:ff9b::7f00:1` would otherwise be a public address that reaches 127.0.0.1."+ ^InetAddress [^InetAddress addr]+ (if (address-in-prefix? addr nat64-well-known-prefix)+ (InetAddress/getByAddress (Arrays/copyOfRange (.getAddress addr) 12 16))+ addr))+(defn public-address?- "True only for globally-routable unicast IP addresses (rejects loopback, link-local, site-local,- any-local, multicast, IPv6 unique-local fc00::/7, IPv4 CGNAT 100.64.0.0/10, IPv4 \"this network\"- 0.0.0.0/8, and IPv4 reserved 240.0.0.0/4 -- which includes the 255.255.255.255 broadcast address)."+ "True only for globally reachable unicast IP addresses.++ In addition to the address classes recognized by `java.net.InetAddress`, this rejects the non-global blocks in the+ IANA IPv4 and IPv6 Special-Purpose Address Registries while preserving their more-specific globally reachable+ entries. IPv6 addresses outside the *allocated* part of the global-unicast space are rejected unless a registry+ marks them globally reachable: an internal network can route reserved space.++ A NAT64 address is judged by the IPv4 it translates to; see [[translated-address]]."[^InetAddress addr]- (let [b (.getAddress addr)+ (let [addr (translated-address addr)+ b (.getAddress addr)ipv4? (= 4 (alength b))b0 (bit-and (aget b 0) 0xff)]- (not (or (.isLoopbackAddress addr)- (.isLinkLocalAddress addr)- (.isSiteLocalAddress addr)- (.isAnyLocalAddress addr)- (.isMulticastAddress addr)- (and (instance? Inet6Address addr) ; IPv6 unique-local fc00::/7- (= 0xfc (bit-and (aget b 0) 0xfe)))- (and ipv4? ; IPv4 CGNAT 100.64.0.0/10- (= 100 b0)- (<= 64 (bit-and (aget b 1) 0xff) 127))- (and ipv4? (zero? b0)) ; IPv4 "this network" 0.0.0.0/8- (and ipv4? (<= 240 b0)))))) ; IPv4 reserved 240.0.0.0/4 + broadcast+ (and (not (or (.isLoopbackAddress addr)+ (.isLinkLocalAddress addr)+ (.isSiteLocalAddress addr)+ (.isAnyLocalAddress addr)+ (.isMulticastAddress addr)+ (and (instance? Inet6Address addr) ; IPv6 unique-local fc00::/7+ (= 0xfc (bit-and (aget b 0) 0xfe)))+ (and ipv4? ; IPv4 CGNAT 100.64.0.0/10+ (= 100 b0)+ (<= 64 (bit-and (aget b 1) 0xff) 127))+ (and ipv4? (zero? b0)) ; IPv4 "this network" 0.0.0.0/8+ (and ipv4? (<= 240 b0)))) ; IPv4 reserved 240.0.0.0/4 + broadcast+ (or (some #(address-in-prefix? addr %) globally-reachable-special-prefixes)+ (and (or ipv4?+ (= 0x20 (bit-and b0 0xe0))) ; IPv6 global unicast 2000::/3+ (not-any? #(address-in-prefix? addr %) non-global-prefixes))))))(defn- private-address?"True for addresses that are private but may be intentionally reachable from a self-hosted deployment."[^InetAddress addr]- (let [b (.getAddress addr)+ (let [addr (translated-address addr)+ b (.getAddress addr)ipv4? (= 4 (alength b))](or (.isSiteLocalAddress addr)(and (instance? Inet6Address addr) ; IPv6 unique-local fc00::/7@@ -86,7 +185,7 @@(defn address-allowed-for-network-policy?"Whether `addr` is allowed by `policy`.- `:external-only` allows only globally routable public addresses.+ `:external-only` allows only globally reachable public addresses.`:allow-private` adds private, unique-local and carrier-grade NAT addresses.`:loopback-and-private` allows *only* loopback plus those same private ranges`:allow-all` imposes no address restriction."@@ -176,6 +275,28 @@(throw (ex-info "Refusing to connect to a non-permitted network address"{:ssrf true :policy policy :host host}))))))))+(def ^:dynamic *proxy-selector*+ "The `ProxySelector` [[jvm-proxied-url?]] asks. nil reads `ProxySelector/getDefault` at call time, which is what+ Apache HttpClient's route planner does; tests bind it rather than installing a selector process-wide."+ nil)++(defn jvm-proxied-url?+ "Whether the JVM's proxy configuration -- `-Dhttps.proxyHost` and friends, or `java.net.useSystemProxies` -- puts a+ proxy in front of `url`.++ This decides whether a `:dns-resolver` can enforce anything. clj-http's default route planner honours the JVM+ proxy settings, and the connection it then opens is to the *proxy*: the resolver is handed the proxy's hostname,+ and the target is resolved by the proxy, out of reach. A caller enforcing a network policy has to check the target+ host itself in that case."+ [url]+ (try+ (let [parsed (URL. (str url))+ target (HttpHost. (.getHost parsed) (.getPort parsed) (.getProtocol parsed))+ planner (SystemDefaultRoutePlanner. ^ProxySelector *proxy-selector*)]+ ;; Use the client's route planner: DIRECT takes precedence over later proxies, and SOCKS entries are ignored.+ (some? (.getProxyHost (.determineRoute planner target (BasicHttpRequest. "GET" "/") (BasicHttpContext.)))))+ (catch Exception _ false)))+(defn safe-url?"True if `url` is safe to fetch from untrusted input: HTTPS scheme, no userinfo, and a real DNShostname (not an IP literal, not localhost/metadata/internal). Note this is a cheap pre-check;diff --git a/test/metabase/geojson/settings_test.clj b/test/metabase/geojson/settings_test.cljindex bdfee4ccd5b..d2909299393 100644--- a/test/metabase/geojson/settings_test.clj+++ b/test/metabase/geojson/settings_test.clj@@ -53,6 +53,10 @@"http://[::]" false"http://100.64.0.1" false"http://[fc00::1]" false+ ;; Special-purpose addresses that are not globally reachable+ "http://192.0.2.0" false+ "http://198.18.0.1" false+ "http://[2001:db8::1]" false;; alternate IPv4 encodings (hex, octal, integer)"http://0xa9fea9fe" false"https://0xa9fea9fe" false@@ -71,7 +75,7 @@"http://example.com/" true"https://example.com/" true"http://example.com/rivendell.json" true- "http://192.0.2.0" true+ "http://8.8.8.8" true;; this following test flakes in CI for unknown reasons;;"http://0xc0000200" true;; Classpath resources are NOT valid when env var is not setdiff --git a/test/metabase/llm/anthropic_test.clj b/test/metabase/llm/anthropic_test.cljindex 0769dae6a06..a91c75a639e 100644--- a/test/metabase/llm/anthropic_test.clj+++ b/test/metabase/llm/anthropic_test.clj@@ -5,7 +5,8 @@[metabase.config.core :as config][metabase.llm.anthropic :as anthropic][metabase.llm.settings :as llm.settings]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.http :as u.http]))(set! *warn-on-reflection* true)@@ -119,11 +120,55 @@(let [mock-response {:body {:content [{:type "tool_use" :name "generate_sql" :input {:sql "SELECT 1"}}]:usage {:input_tokens 1 :output_tokens 1}}}](with-redefs [config/is-e2e? true]+ ;; the e2e runner pins the network policy the same way, since the mock LLM server is on localhost+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"+ llm-anthropic-api-base-url "http://localhost:6123"]+ (mt/with-dynamic-fn-redefs [http/post (constantly mock-response)]+ (is (=? {:result {:sql "SELECT 1"}}+ (anthropic/chat-completion {:messages [{:role "user" :content "test"}]}))))))))))++(deftest chat-completion-network-policy-test+ (testing "a base URL on a network llm-allowed-networks forbids is refused when the connection resolves it"+ ;; the URL is redefined rather than set, the way one saved before the policy was tightened would be stored;+ ;; the mock stands in for clj-http far enough to run the request's `:dns-resolver` on the host+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-dynamic-fn-redefs [llm.settings/llm-anthropic-api-key (constantly "sk-ant-test-key")+ llm.settings/llm-anthropic-api-base-url (constantly "http://127.0.0.1:9")+ http/post (fn [url opts]+ (.resolve ^org.apache.http.conn.DnsResolver (:dns-resolver opts)+ (u.http/->hostname url))+ (is false "the resolver should have refused the address"))]+ (is (=? {:status-code 400 :status 400 :error-code :llm-host-not-allowed :llm-host "127.0.0.1"}+ (try (anthropic/chat-completion {:messages [{:role "user" :content "test"}]})+ (catch clojure.lang.ExceptionInfo e (ex-data e))))))))+ (testing "a permitted base URL goes out with the policy-enforcing DNS resolver on the connection"+ (let [captured (atom nil)+ mock-response {:body {:content [{:type "tool_use" :name "generate_sql" :input {:sql "SELECT 1"}}]+ :usage {:input_tokens 1 :output_tokens 1}}}]+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"](mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"- llm-anthropic-api-base-url "http://localhost:6123"]- (mt/with-dynamic-fn-redefs [http/post (constantly mock-response)]+ llm-anthropic-api-base-url "https://8.8.8.8"]+ (mt/with-dynamic-fn-redefs [http/post (fn [_url opts] (reset! captured opts) mock-response)](is (=? {:result {:sql "SELECT 1"}}- (anthropic/chat-completion {:messages [{:role "user" :content "test"}]})))))))))+ (anthropic/chat-completion {:messages [{:role "user" :content "test"}]})))+ (is (= :none (:redirect-strategy @captured)))+ (is (instance? org.apache.http.conn.DnsResolver (:dns-resolver @captured))))))+ (testing "a connection-time DNS policy rejection has the same 400 shape as the upfront check"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"+ llm-anthropic-api-base-url "https://8.8.8.8"]+ (mt/with-dynamic-fn-redefs [http/post (fn [& _] (throw (ex-info "blocked" {:ssrf true})))]+ (is (=? {:status-code 400 :api-error true :error-code :llm-host-not-allowed}+ (try (anthropic/chat-completion {:messages [{:role "user" :content "test"}]})+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))))))+ (testing "and under :allow-all on clj-http's default resolver"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"+ llm-anthropic-api-base-url "http://127.0.0.1:9"]+ (mt/with-dynamic-fn-redefs [http/post (fn [_url opts] (reset! captured opts) mock-response)]+ (anthropic/chat-completion {:messages [{:role "user" :content "test"}]})+ (is (not (contains? @captured :dns-resolver))))))))))(deftest chat-completion-returns-usage-test(testing "chat-completion returns result, usage, and duration"diff --git a/test/metabase/llm/api/provider_test.clj b/test/metabase/llm/api/provider_test.cljindex 74d617e6348..7c8d521d211 100644--- a/test/metabase/llm/api/provider_test.clj+++ b/test/metabase/llm/api/provider_test.clj@@ -11,7 +11,8 @@[metabase.settings.core :as setting][metabase.settings.models.setting.cache :as setting.cache][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]))+ [metabase.test.fixtures :as fixtures]+ [metabase.util.json :as json]))(use-fixtures :once (fixtures/initialize :db))@@ -495,6 +496,32 @@{:type "evilai" :config {:api-key "whatever"}}))))(is (= [] (llm.provider/connections))))))+(deftest create-rejects-a-base-url-on-a-blocked-network-before-calling-the-provider-test+ (testing (str "verifying credentials fetches the provider's model catalog from the base URL, so a base URL "+ "on a network the policy forbids is refused before that request is made")+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-temporary-setting-values [llm-providers []]+ (mt/with-dynamic-fn-redefs [metabot.self/list-models+ (fn [& _] (is false "should reject before verifying credentials"))]+ (let [create #(mt/user-http-request :crowberto :post 400 "llm/providers"+ {:type "anthropic"+ :config {:api-key "sk-ant-valid"+ :base-url "http://127.0.0.1:9"}})]+ (testing "self-hosted, the message names the setting to change"+ (mt/with-premium-features #{}+ (is (=? {:message (str "The base URL host 127.0.0.1 is on a network Metabase is not allowed to "+ "connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-private for a server on "+ "your private network, or allow-all for one on this machine.")+ :field "base-url"}+ (create)))))+ (testing "on Cloud there is no setting to change, and the message says so"+ (mt/with-premium-features #{:hosting}+ (is (=? {:message (str "The base URL host 127.0.0.1 is not permitted by Metabase Cloud's LLM network policy. "+ "Use an LLM provider on the public internet.")+ :field "base-url"}+ (create))))))+ (is (= [] (llm.provider/connections))))))))+(deftest create-suffixes-a-colliding-key-test(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-first"})]](mt/with-dynamic-fn-redefs [metabot.self/list-models (constantly {:models []})]@@ -1246,3 +1273,36 @@(mt/user-http-request :crowberto :put 200 "llm/providers/anthropic"{:config {:api-key "sk-ant-rotated"}}))(is (= {:api-key "sk-ant-rotated"} (stored-config "anthropic")))))))++(deftest settings-api-cannot-store-a-base-url-on-a-blocked-network-test+ (testing (str "the connection list is a setting in its own right, so writing the raw JSON through the settings "+ "API has to be refused the way the connection endpoints refuse it")+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-temporary-setting-values [llm-providers []]+ (let [conn #(connection "vllm" "vllm" {:base-url %})]+ (is (=? {:message #".*127\.0\.0\.1 is on a network.*"+ :field "base-url"}+ (mt/user-http-request :crowberto :put 400 "setting/llm-providers"+ {:value [(conn "http://127.0.0.1:8000/v1")]})))+ (is (= [] (vec (llm.provider/stored-connections))))+ (testing "a base URL the policy permits still saves"+ (mt/user-http-request :crowberto :put 204 "setting/llm-providers"+ {:value [(conn "https://8.8.8.8/v1")]})+ (is (= [(conn "https://8.8.8.8/v1")] (vec (llm.provider/stored-connections)))))+ (testing "a base URL stored before the check does not make its connection unwritable"+ (let [grandfathered (assoc-in (conn "http://127.0.0.1:8000/v1") [:config :api-key] "sk-old")]+ (mt/with-temporary-raw-setting-values [llm-providers (json/encode [grandfathered])]+ (testing "another connection can still be added"+ (mt/user-http-request :crowberto :put 204 "setting/llm-providers"+ {:value [grandfathered+ (connection "anthropic" "anthropic" {:api-key "sk-ant-valid"})]})+ (is (= ["vllm" "anthropic"] (map :key (llm.provider/stored-connections)))))+ (testing "and its own API key can still be rotated"+ (mt/user-http-request :crowberto :put 204 "setting/llm-providers"+ {:value [(assoc-in grandfathered [:config :api-key] "sk-new")]})+ (is (= "sk-new" (get-in (first (llm.provider/stored-connections)) [:config :api-key]))))+ (testing "but changing the base URL itself is still checked"+ (is (=? {:field "base-url"}+ (mt/user-http-request :crowberto :put 400 "setting/llm-providers"+ {:value [(assoc-in grandfathered+ [:config :base-url] "http://10.0.0.1/v1")]}))))))))))))diff --git a/test/metabase/llm/provider_test.clj b/test/metabase/llm/provider_test.cljindex f370abb8bb0..5f4e573b6d6 100644--- a/test/metabase/llm/provider_test.clj+++ b/test/metabase/llm/provider_test.clj@@ -277,6 +277,25 @@(llm.provider/validate-config! "google" {:service-account-key "{\"type\":\"service_account\"}":location "US Central"})))))+(deftest validate-config!-base-url-network-policy-test+ (testing "every provider's base URL is checked against llm-allowed-networks"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (doseq [[type config] [["anthropic" {:api-key "sk-ant-valid"}]+ ["openai" {:api-key "sk-valid"}]+ ["vllm" {}]+ ["azure" {:api-key "azure-key" :deployment-name "gpt-4.1-mini"}]]]+ (testing type+ (is (=? {:status-code 400 :field :base-url}+ (try (llm.provider/validate-config! type (assoc config :base-url "http://127.0.0.1:8000"))+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))+ (is (nil? (llm.provider/validate-config! type (assoc config :base-url "https://8.8.8.8/v1"))))))))+ (testing "under :allow-all a loopback base URL is fine, but it still has to be an http(s) URL"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (nil? (llm.provider/validate-config! "vllm" {:base-url "http://127.0.0.1:8000/v1"})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"must start with http"+ (llm.provider/validate-config! "vllm" {:base-url "127.0.0.1:8000/v1"}))))))+(deftest validate-config!-select-options-test(testing "a select field's value must be one of its options"(is (thrown-with-msg?diff --git a/test/metabase/llm/settings_test.clj b/test/metabase/llm/settings_test.cljindex b803a8f7350..1f05af6039e 100644--- a/test/metabase/llm/settings_test.clj+++ b/test/metabase/llm/settings_test.clj@@ -4,7 +4,10 @@[metabase.config.core :as config][metabase.llm.settings :as llm.settings][metabase.settings.core :as setting]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.http :as u.http])+ (:import+ (java.net InetSocketAddress Proxy Proxy$Type ProxySelector)))(set! *warn-on-reflection* true)@@ -196,6 +199,208 @@(is (nil? (llm.settings/assert-llm-host-allowed! nil)))(is (nil? (llm.settings/assert-llm-host-allowed! "")))))))+;;; ----------------------------------------- llm-allowed-networks Tests -----------------------------------------++(deftest llm-allowed-networks-default-test+ ;; the raw binding clears any stored value, so the default is what is under test+ (mt/with-temporary-raw-setting-values [llm-allowed-networks nil]+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks nil]+ (testing "with nothing configured only public addresses are allowed, hosted or not"+ (mt/with-premium-features #{}+ (is (= :external-only (llm.settings/llm-allowed-networks))))+ (mt/with-premium-features #{:hosting}+ (is (= :external-only (llm.settings/llm-allowed-networks))))))+ (testing "the environment can loosen it"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (= :allow-all (llm.settings/llm-allowed-networks))))+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-private"]+ (is (= :allow-private (llm.settings/llm-allowed-networks)))))+ (testing "a value that is not one of the policies fails closed"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow_all"]+ (is (= :external-only (llm.settings/llm-allowed-networks)))))+ (testing "it is not settable: nobody loosens it through the API"+ (is (thrown? Exception (setting/set! :llm-allowed-networks :allow-all)))))+ (testing "a value that reached the application database some other way is ignored"+ (mt/with-temporary-raw-setting-values [llm-allowed-networks "allow-all"]+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks nil]+ (is (= :external-only (llm.settings/llm-allowed-networks)))))))++(deftest network-policy-floor-test+ (testing "a deployment-controlled endpoint's floor can only loosen the configured policy"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (is (= :external-only (llm.settings/network-policy)))+ (is (= :external-only (llm.settings/network-policy nil)))+ (is (= :allow-private (llm.settings/network-policy :allow-private))))+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (= :allow-all (llm.settings/network-policy :allow-private))))))++(deftest llm-url-problem-test+ ;; IP literals throughout: `host-allowed-for-network-policy?` resolves hostnames through real DNS+ (testing "under :external-only, internal addresses are refused and public ones are not"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (doseq [url ["http://127.0.0.1:8000/v1"+ "http://[::1]:8000/v1"+ "http://169.254.169.254/latest/meta-data/"+ "http://10.0.0.1/v1"+ "http://192.168.1.1/v1"]]+ (is (some? (llm.settings/llm-url-problem url)) url))+ (is (nil? (llm.settings/llm-url-problem "https://8.8.8.8/v1")))))+ (testing "the message tells a self-hosted operator which setting to change, and a Cloud admin that there is none"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-premium-features #{}+ (is (= (str "The base URL host 10.0.0.1 is on a network Metabase is not allowed to connect to. "+ "Set MB_LLM_ALLOWED_NETWORKS=allow-private for a server on your private network, "+ "or allow-all for one on this machine.")+ (llm.settings/llm-url-problem "http://10.0.0.1/v1"))))+ (mt/with-premium-features #{:hosting}+ (is (= (str "The base URL host 10.0.0.1 is not permitted by Metabase Cloud's LLM network policy. "+ "Use an LLM provider on the public internet.")+ (llm.settings/llm-url-problem "http://10.0.0.1/v1"))))))+ (testing "under :allow-private, private networks pass but loopback and link-local still do not"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-private"]+ (is (nil? (llm.settings/llm-url-problem "http://10.0.0.1/v1")))+ (is (some? (llm.settings/llm-url-problem "http://127.0.0.1:8000/v1")))+ (is (some? (llm.settings/llm-url-problem "http://169.254.169.254/")))))+ (testing "the two-argument form takes the policy to enforce"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (is (nil? (llm.settings/llm-url-problem :allow-private "http://10.0.0.1/v1")))+ (is (some? (llm.settings/llm-url-problem :allow-private "http://127.0.0.1/v1")))))+ (testing "under :allow-all, anything reachable goes"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (nil? (llm.settings/llm-url-problem "http://127.0.0.1:8000/v1")))+ (is (nil? (llm.settings/llm-url-problem "http://169.254.169.254/")))))+ (testing "a URL that is not http(s), or has no host, is refused under any policy"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (doseq [url ["api.openai.com/v1" "ftp://8.8.8.8/v1" "file:///etc/passwd" "http://" "not a url"]]+ (is (re-find #"must start with http" (llm.settings/llm-url-problem url)) url))))+ (testing "credentials in the URL are refused under any policy: they would ride along into error messages"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (re-find #"username or password" (llm.settings/llm-url-problem "https://svc:[email protected]/v1")))))+ (testing "a blank URL is not a problem here: the not-configured handling covers it"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (is (nil? (llm.settings/llm-url-problem nil)))+ (is (nil? (llm.settings/llm-url-problem " "))))))++(deftest llm-request-opts-test+ ;; IP literals throughout: the resolver goes through real DNS+ (let [resolver (fn [& args] (:dns-resolver (apply llm.settings/llm-request-opts args)))]+ (testing "redirects are disabled under every network policy"+ (doseq [policy ["external-only" "allow-private" "allow-all"]]+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks policy]+ (is (= :none (:redirect-strategy (llm.settings/llm-request-opts "https://8.8.8.8/v1")))))))+ (testing "under :external-only the request gets the policy resolver; no lookup happens here"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (is (instance? org.apache.http.conn.DnsResolver (resolver "http://127.0.0.1:8000/v1")))+ (testing "and it is the resolver that refuses an address the policy does not permit"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"non-permitted"+ (.resolve ^org.apache.http.conn.DnsResolver (resolver "http://127.0.0.1:8000/v1") "127.0.0.1")))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"non-permitted"+ (.resolve ^org.apache.http.conn.DnsResolver (resolver "http://10.0.0.1/v1") "10.0.0.1"))))+ (testing "a floor loosens it for a deployment-controlled endpoint"+ (is (= 1 (alength ^"[Ljava.net.InetAddress;"+ (.resolve ^org.apache.http.conn.DnsResolver (resolver :allow-private "http://10.0.0.1/v1")+ "10.0.0.1")))))))+ (testing "under :allow-all there is no policy resolver"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (is (= {:redirect-strategy :none}+ (llm.settings/llm-request-opts "http://127.0.0.1:8000/v1")))))+ (testing "a URL that is not usable at all is refused up front, naming the host but not what else it carried"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (doseq [url ["https://svc:[email protected]/v1" "8.8.8.8/v1"]]+ (is (=? {:status-code 400+ :status 400+ :api-error true+ :error-code :llm-host-not-allowed+ :llm-host "8.8.8.8"}+ (try (llm.settings/llm-request-opts url)+ (catch clojure.lang.ExceptionInfo e (ex-data e))))+ url))))))++(defn- proxy-selector+ "A `ProxySelector` that answers `proxies` for every URI, the way a JVM configured with `-Dhttps.proxyHost` does."+ ^ProxySelector [proxies]+ (proxy [ProxySelector] []+ (select [_uri] proxies)+ (connectFailed [_uri _sa _ioe] nil)))++(deftest llm-request-opts-behind-a-jvm-proxy-test+ (testing (str "clj-http routes through a JVM-configured proxy, so the connection is opened to the proxy and a "+ "`:dns-resolver` never sees the target: the target host is checked up front instead")+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (binding [u.http/*proxy-selector*+ (proxy-selector [(Proxy. Proxy$Type/HTTP (InetSocketAddress. "10.0.0.9" 3128))])]+ (testing "the proxy is deployment configuration, so a private one is not judged by the policy"+ (is (= {:redirect-strategy :none} (llm.settings/llm-request-opts "https://8.8.8.8/v1"))))+ (testing "and the target behind it still is"+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed :llm-host "10.0.0.1"}+ (try (llm.settings/llm-request-opts "http://10.0.0.1/v1")+ (catch clojure.lang.ExceptionInfo e (ex-data e))))))+ (testing "proxy-only DNS remains supported without following redirects"+ (let [looked-up (atom [])]+ (mt/with-dynamic-fn-redefs [u.http/host->inet-addresses+ (fn [host]+ (swap! looked-up conj host)+ nil)]+ (is (= {:redirect-strategy :none}+ (llm.settings/llm-request-opts "https://proxy-only.example/v1")))+ (is (= ["proxy-only.example"] @looked-up)))))+ (testing "a floor loosens the target check for a deployment-controlled endpoint"+ (is (= {:redirect-strategy :none}+ (llm.settings/llm-request-opts :allow-private "http://10.0.0.1/v1")))))+ (testing "with the proxy selector answering DIRECT, the resolver does the enforcing as before"+ (binding [u.http/*proxy-selector* (proxy-selector [Proxy/NO_PROXY])]+ (is (instance? org.apache.http.conn.DnsResolver+ (:dns-resolver (llm.settings/llm-request-opts "http://10.0.0.1/v1")))))))))++(deftest llm-request-opts-retains-resolver-for-direct-proxy-routes-test+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (doseq [proxies [[Proxy/NO_PROXY (Proxy. Proxy$Type/HTTP (InetSocketAddress. "10.0.0.9" 3128))]+ [(Proxy. Proxy$Type/SOCKS (InetSocketAddress. "10.0.0.9" 1080))]]]+ (binding [u.http/*proxy-selector* (proxy-selector proxies)]+ (let [resolver (:dns-resolver (llm.settings/llm-request-opts "http://127.0.0.1/v1"))]+ (is (instance? org.apache.http.conn.DnsResolver resolver))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Refusing to connect"+ (.resolve ^org.apache.http.conn.DnsResolver resolver "127.0.0.1"))))))))++(deftest connection-time-network-policy-error-test+ (testing "direct and wrapped DNS policy rejections are recognized and translated to the URL-validation shape"+ (doseq [cause [(ex-info "blocked address" {:ssrf true})+ (ex-info "HTTP client wrapper" {} (ex-info "blocked address" {:ssrf true}))]]+ (is (true? (llm.settings/llm-network-policy-error? cause)))+ (is (=? {:status-code 400+ :status 400+ :api-error true+ :error-code :llm-host-not-allowed+ :llm-host "rebound.example"}+ (try (llm.settings/rethrow-if-llm-network-policy-error!+ cause "https://rebound.example/v1")+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))))+ (testing "the message names a policy that would actually change the outcome"+ (mt/with-premium-features #{}+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (testing "under external-only, allow-private is the next step"+ (is (re-find #"allow-private for a server on your private network"+ (llm.settings/llm-url-problem "http://10.0.0.1/v1"))))+ (testing "under a deployment endpoint's allow-private floor, it is already in force -- name allow-all"+ (is (re-find #"Set MB_LLM_ALLOWED_NETWORKS=allow-all"+ (llm.settings/llm-url-problem :allow-private "http://127.0.0.1/v1")))+ (is (=? {:llm-host "127.0.0.1"}+ (try (llm.settings/rethrow-if-llm-network-policy-error!+ (ex-info "blocked" {:ssrf true :policy :allow-private})+ "http://127.0.0.1/v1")+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))+ (is (re-find #"Set MB_LLM_ALLOWED_NETWORKS=allow-all"+ (try (llm.settings/rethrow-if-llm-network-policy-error!+ (ex-info "blocked" {:ssrf true :policy :allow-private})+ "http://127.0.0.1/v1")+ (catch clojure.lang.ExceptionInfo e (ex-message e)))))))))+ (testing "unrelated failures pass through the recognizer"+ (let [e (ex-info "provider down" {:status 503})]+ (is (false? (llm.settings/llm-network-policy-error? e)))+ (is (nil? (llm.settings/rethrow-if-llm-network-policy-error! e "https://example.com"))))))+;;; ------------------------------------------- Settings Defaults Tests -------------------------------------------(deftest llm-max-tokens-testdiff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 4afd1c48b8a..6fbea809f09 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -253,52 +253,54 @@(try(mt/test-helpers-set-global-values!(search.tu/with-index-disabled- (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]- metabot.settings/llm-metabot-provider test-provider]- (let [real-llm-request self.core/request]- (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)- conversation-title/ensure-title! (constantly {:status :missing})- ;; The fake LLM server gzips whenever the caller accepts it, and clj-http- ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.- self.core/request (fn [auth req]- (real-llm-request auth (assoc req :decompress-body false)))- metabot.context/create-context (fn [ctx & _] ctx)- metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]- (reset! stored-parts parts)- (reset! stored-kwargs (apply hash-map kwargs)))- sr/async-cancellation-poll-interval-ms 5]- (testing "Closing stream body tears down the pipeline and persists the aborted turn"- (reset! cnt total-chunks)- (reset! stored-parts nil)- (reset! stored-kwargs nil)- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (let [conversation-id (str (random-uuid))- response (mt/user-real-request-full-response- :rasta :post 202 "metabot/agent-streaming"- {:request-options {:as :stream- :decompress-body false}}- {:message "Test closure"- :context {}- :conversation_id conversation-id- :state {}})]- (.read ^java.io.InputStream (:body response)) ;; start the handler- ;; Close the underlying client, not the body stream: closing the body would- ;; make clj-http drain the (now chunked) response to completion, which looks- ;; like a normal finish rather than a disconnect. Closing the client aborts- ;; the connection, which is what the server's cancel loop detects.- (.close ^java.io.Closeable (:http-client response))- (u/poll {:thunk #(deref stored-parts)- :done? some?- :interval-ms 10- :timeout-ms 3000})- (is (some? @stored-parts)- "finalize-assistant-turn! was called even though the client disconnected")- (is (false? (:finished? @stored-kwargs))- "the finalized turn is marked :finished? false — the cancel was detected")- (is (= 2 (count (t2/select :model/MetabotMessage- :conversation_id conversation-id)))- "start-turn! inserted exactly user + placeholder; no extra row from finalize")))))))))+ ;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]+ metabot.settings/llm-metabot-provider test-provider]+ (let [real-llm-request self.core/request]+ (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)+ conversation-title/ensure-title! (constantly {:status :missing})+ ;; The fake LLM server gzips whenever the caller accepts it, and clj-http+ ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.+ self.core/request (fn [auth req]+ (real-llm-request auth (assoc req :decompress-body false)))+ metabot.context/create-context (fn [ctx & _] ctx)+ metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]+ (reset! stored-parts parts)+ (reset! stored-kwargs (apply hash-map kwargs)))+ sr/async-cancellation-poll-interval-ms 5]+ (testing "Closing stream body tears down the pipeline and persists the aborted turn"+ (reset! cnt total-chunks)+ (reset! stored-parts nil)+ (reset! stored-kwargs nil)+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (let [conversation-id (str (random-uuid))+ response (mt/user-real-request-full-response+ :rasta :post 202 "metabot/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id conversation-id+ :state {}})]+ (.read ^java.io.InputStream (:body response)) ;; start the handler+ ;; Close the underlying client, not the body stream: closing the body would+ ;; make clj-http drain the (now chunked) response to completion, which looks+ ;; like a normal finish rather than a disconnect. Closing the client aborts+ ;; the connection, which is what the server's cancel loop detects.+ (.close ^java.io.Closeable (:http-client response))+ (u/poll {:thunk #(deref stored-parts)+ :done? some?+ :interval-ms 10+ :timeout-ms 3000})+ (is (some? @stored-parts)+ "finalize-assistant-turn! was called even though the client disconnected")+ (is (false? (:finished? @stored-kwargs))+ "the finalized turn is marked :finished? false — the cancel was detected")+ (is (= 2 (count (t2/select :model/MetabotMessage+ :conversation_id conversation-id)))+ "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))))(finally(.stop llm-server))))))diff --git a/test/metabase/metabot/self/google_test.clj b/test/metabase/metabot/self/google_test.cljindex 8fd58e4a772..c4388778f8d 100644--- a/test/metabase/metabot/self/google_test.clj+++ b/test/metabase/metabot/self/google_test.clj@@ -31,15 +31,17 @@"\n-----END PRIVATE KEY-----\n"))))(defn- test-service-account-json- "A structurally valid service account key JSON for `project-id`."- [project-id]- (json/encode {:type "service_account"- :project_id project-id- :private_key_id "test-key-id"- :private_key @test-private-key-pem- :client_email (str "metabot-test@" project-id ".iam.gserviceaccount.com")- :client_id "123456789012345678901"- :token_uri "https://oauth2.googleapis.com/token"}))+ "A structurally valid service account key JSON for `project-id`; Google's `token_uri` unless given one."+ ([project-id]+ (test-service-account-json project-id "https://oauth2.googleapis.com/token"))+ ([project-id token-uri]+ (json/encode (cond-> {:type "service_account"+ :project_id project-id+ :private_key_id "test-key-id"+ :private_key @test-private-key-pem+ :client_email (str "metabot-test@" project-id ".iam.gserviceaccount.com")+ :client_id "123456789012345678901"}+ token-uri (assoc :token_uri token-uri)))));;; The adapter serves a request from the credentials of the connection behind it. These tests bind the;;; `llm-google-*` settings, which are the environment-configured form of that connection, so the calls below@@ -250,6 +252,36 @@:error-code :invalid-project-id}(ex-data e)))))))+(deftest service-account-key-token-uri-pinned-test+ (testing (str "the credential library posts to the key's token_uri to mint an access token, before any "+ "request the network policy guards, so a key naming anything but Google's OAuth endpoint is refused")+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]+ (doseq [token-uri ["http://169.254.169.254/latest/meta-data/"+ "https://127.0.0.1/token"+ "https://evil.example.com/token"+ "http://oauth2.googleapis.com/token"+ ;; the whole URI is pinned, not just its origin+ "https://accounts.google.com/not-a-token-endpoint"+ "https://oauth2.googleapis.com:8443/token"+ "https://oauth2.googleapis.com/token?x=1"+ "https://oauth2.googleapis.com/token/../../evil"]]+ (testing token-uri+ (is (=? {:api-error true+ :status-code 400+ :error-code :invalid-service-account-key}+ (try (list-models {:model "google/gemini-3.5-flash"+ :credentials {:service-account-key (test-service-account-json "p" token-uri)+ :project-id "my-project"}})+ nil+ (catch Exception e (ex-data e)))))))))+ (testing "Google's current and legacy token endpoints, and the library default when the key names none, parse"+ (doseq [token-uri ["https://oauth2.googleapis.com/token"+ "https://accounts.google.com/o/oauth2/token"+ nil]]+ (testing (pr-str token-uri)+ (is (instance? ServiceAccountCredentials+ (#'google/parse-service-account-credentials (test-service-account-json "p" token-uri))))))))+(deftest google-raw-service-account-json-project-id-rejected-test(testing "the project ID carried by a service account key JSON is validated as well"(let [sa-key (test-service-account-json "Not-A-Project")]diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex dc7b5b7ab19..be129102a3e 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -24,6 +24,7 @@[metabase.metabot.usage :as usage][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.http :as u.http][metabase.util.json :as json][metabase.util.log.capture :as log.capture][metabase.util.malli :as mu]@@ -178,6 +179,70 @@(is (= 100 (:connection-timeout @captured)))(is (= 200 (:socket-timeout @captured)))))))))+(deftest request-enforces-llm-allowed-networks-test+ ;; IP literals throughout: the resolver goes through real DNS. `resolving` stands in for clj-http far enough to+ ;; run the request's `:dns-resolver` on its host, which is where the policy is enforced.+ (let [captured (atom nil)+ resolving (fn [{:keys [url] :as opts}]+ (some-> ^org.apache.http.conn.DnsResolver (:dns-resolver opts) (.resolve (u.http/->hostname url)))+ (reset! captured opts)+ {:status 200 :body ""})+ req {:method :get :url "/v1/models"}+ rejected (fn [auth]+ (try (self.core/request auth req)+ nil+ (catch clojure.lang.ExceptionInfo e (ex-data e))))]+ (testing "under :external-only"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "external-only"]+ (mt/with-dynamic-fn-redefs [http/request resolving]+ (testing "a base URL on an internal network is refused when the connection resolves it"+ (is (=? {:status-code 400+ :status 400+ :api-error true+ :error-code :llm-host-not-allowed+ :llm-host "127.0.0.1"}+ (rejected {:url "http://127.0.0.1:9" :headers {}}))))+ (testing "a public base URL goes out with the policy resolver on the connection"+ (self.core/request {:url "https://8.8.8.8" :headers {}} req)+ (is (= "https://8.8.8.8/v1/models" (:url @captured)))+ (is (= :none (:redirect-strategy @captured)))+ (is (instance? org.apache.http.conn.DnsResolver (:dns-resolver @captured))))+ (testing "the managed AI proxy's floor allows private addresses under the default policy"+ (self.core/request {:url "http://10.0.0.1:9" :headers {} :network-policy-floor :allow-private} req)+ (is (= "http://10.0.0.1:9/v1/models" (:url @captured))))+ (testing "but still refuses loopback"+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed}+ (rejected {:url "http://127.0.0.1:9" :headers {} :network-policy-floor :allow-private})))))+ (testing "a URL that is not http(s) is refused before any request is made"+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (is false "http/request should not be called"))]+ (is (=? {:status-code 400 :error-code :llm-host-not-allowed :llm-host "8.8.8.8"}+ (rejected {:url "8.8.8.8" :headers {}})))))))+ (testing "under :allow-all an internal base URL goes out on clj-http's default resolver"+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-dynamic-fn-redefs [http/request resolving]+ (self.core/request {:url "http://127.0.0.1:9" :headers {}} req)+ (is (= "http://127.0.0.1:9/v1/models" (:url @captured)))+ (is (not (contains? @captured :dns-resolver)))+ (testing "and the AI proxy's floor does not tighten that: a proxy on this machine is reachable"+ (self.core/request {:url "http://127.0.0.1:9" :headers {} :network-policy-floor :allow-private} req)+ (is (= "http://127.0.0.1:9/v1/models" (:url @captured)))+ (is (not (contains? @captured :dns-resolver)))))))))++(deftest resolve-auth-tags-proxy-auth-test+ (mt/with-premium-features #{:metabot-v3}+ (testing "an environment-supplied proxy URL carries the private-network floor"+ (mt/with-temp-env-var-value! [mb-llm-proxy-base-url "http://proxy.internal/"]+ (is (=? {:url "http://proxy.internal/anthropic"+ :network-policy-floor :allow-private}+ (self.core/resolve-auth "anthropic" "Anthropic" {:url "https://api.anthropic.com"} true)))))+ (testing "a stored proxy URL gets the default policy: no floor"+ (mt/with-temporary-setting-values [llm-proxy-base-url "http://proxy.internal/"]+ (let [auth (self.core/resolve-auth "anthropic" "Anthropic" {:url "https://api.anthropic.com"} true)]+ (is (= "http://proxy.internal/anthropic" (:url auth)))+ (is (not (contains? auth :network-policy-floor))))+ (is (= {:url "https://api.anthropic.com"}+ (self.core/resolve-auth "anthropic" "Anthropic" {:url "https://api.anthropic.com"} false)))))))+(deftest call-llm-prompt-cache-key-test(llm.tu/with-default-connections(testing "the conversation id (:session-id) is forwarded as the Mistral prompt_cache_key"diff --git a/test/metabase/premium_features/api_test.clj b/test/metabase/premium_features/api_test.cljindex a00bb4699ab..50e465982b7 100644--- a/test/metabase/premium_features/api_test.clj+++ b/test/metabase/premium_features/api_test.clj@@ -5,7 +5,9 @@[clojure.test :refer :all][metabase.premium-features.core :as premium-features][metabase.premium-features.token-check :as token-check]- [metabase.test :as mt]))+ [metabase.test :as mt])+ (:import+ (clojure.lang ExceptionInfo)))(set! *warn-on-reflection* true)@@ -62,9 +64,9 @@(reset! request* [url request-options]){:status 200})](mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")- (is (= [(str "https://ai-service.example.com/v1/invalidate-token-cache/" token)- {:throw-exceptions false}]- @request*))))))))))+ (let [[url opts] @request*]+ (is (= (str "https://ai-service.example.com/v1/invalidate-token-cache/" token) url))+ (is (false? (:throw-exceptions opts))))))))))))(testing "POST /api/premium-features/token/refresh does not invalidate the AI service cache when it is not configured"(mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)premium-features/premium-embedding-token (constantly "proxy-token")@@ -83,3 +85,32 @@{:request-options {:cookie-store cs}})(let [pf-cookie (get (cookies/get-cookies cs) "metabase.PREMIUM_FEATURES_LAST_UPDATED")](is (some? pf-cookie) "No premium-features-last-updated cookie set"))))))++(deftest token-cache-invalidation-carries-the-llm-network-policy-test+ (testing (str "the invalidation URL carries the instance token in its path, and ai-service-base-url is a stored "+ "setting a superuser can write through the generic settings API, so the request is held to the "+ "LLM network policy like any other")+ (let [token "SOME_RANDOM_TOKEN"+ request* (atom nil)+ refresh! (fn []+ (reset! request* nil)+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")+ (second @request*))]+ (mt/with-premium-features #{:metabase-ai-managed}+ (mt/with-temp-env-var-value! [mb-premium-embedding-token nil]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)+ http/post (fn [url opts] (reset! request* [url opts]) {:status 200})]+ (testing "a stored base URL gets no more than llm-allowed-networks allows"+ (mt/with-temporary-setting-values [ai-service-base-url "http://10.0.0.1/"]+ (let [opts (refresh!)]+ (testing "no redirect can carry the token off to a host of the responder's choosing"+ (is (= :none (:redirect-strategy opts))))+ (is (thrown-with-msg?+ ExceptionInfo #"non-permitted"+ (.resolve ^org.apache.http.conn.DnsResolver (:dns-resolver opts) "10.0.0.1"))))))+ (testing "a base URL the environment supplies is deployment configuration and reaches its own cluster"+ (mt/with-temp-env-var-value! [mb-ai-service-base-url "http://10.0.0.1/"]+ (let [opts (refresh!)]+ (is (= 1 (alength ^"[Ljava.net.InetAddress;"+ (.resolve ^org.apache.http.conn.DnsResolver (:dns-resolver opts) "10.0.0.1"))))))))))))))diff --git a/test/metabase/tiles/settings_test.clj b/test/metabase/tiles/settings_test.cljindex d48573c5cdd..f1c0d078161 100644--- a/test/metabase/tiles/settings_test.clj+++ b/test/metabase/tiles/settings_test.clj@@ -69,7 +69,7 @@(testing "the setter still accepts the templates admins legitimately configure"(mt/with-temporary-setting-values [map-tile-server-url osm-template](doseq [template [osm-template- "http://192.0.2.0/{z}/{x}/{y}.png" ; public IP literal+ "http://8.8.8.8/{z}/{x}/{y}.png" ; public IP literal"https://example.com:8443/{z}/{x}/{y}.png?apikey=SEKRIT""/local/{z}/{x}/{y}.png"]] ; same-origin, browser-only(testing templatediff --git a/test/metabase/util/http_test.clj b/test/metabase/util/http_test.cljindex 4a8b9ab4fd3..c82cd01e3af 100644--- a/test/metabase/util/http_test.clj+++ b/test/metabase/util/http_test.clj@@ -5,14 +5,14 @@(:import(clojure.lang ExceptionInfo)(java.io ByteArrayInputStream)- (java.net InetAddress)+ (java.net InetAddress InetSocketAddress Proxy Proxy$Type ProxySelector)(org.apache.http.conn DnsResolver)(org.apache.http.impl.conn InMemoryDnsResolver)))(set! *warn-on-reflection* true)(deftest ^:parallel host-allowed-external-only-test- (testing "external-only allows only globally-routable addresses"+ (testing "external-only allows only globally reachable addresses"(are [host expected] (= expected (http/host-allowed-for-network-policy? :external-only host))"https://example.com" true"8.8.8.8" true@@ -139,33 +139,107 @@"93.184.216.34""100.63.255.255" ; one below the CGNAT 100.64.0.0/10 range"100.128.0.0" ; one above the CGNAT range+ "192.0.0.9" ; globally reachable exception inside 192.0.0.0/24+ "192.0.0.10"+ "192.31.196.1" ; AS112-v4+ "192.52.193.1" ; AMT+ "192.175.48.1" ; direct delegation AS112 service+ "198.17.255.255" ; one below the benchmarking range+ "198.20.0.0" ; one above the benchmarking range+ "64:ff9b::808:808" ; NAT64 well-known prefix, translating to 8.8.8.8+ "2000::1" ; bottom of the allocated global-unicast space 2000::/3+ "2c0f:ffff::1" ; top of the allocated global-unicast space+ "2001:1::1" ; globally reachable exceptions inside 2001::/23+ "2001:1::2"+ "2001:1::3"+ "2001:3::1"+ "2001:4:112::1"+ "2001:20::1"+ "2001:30::1"+ "2001:200::1" ; first /32 beyond 2001::/23+ "2620:4f:8000::1" ; direct delegation AS112 service"2606:4700:4700::1111"]) ; public IPv6+(def ^:private non-global-special-ips+ ["192.0.0.0" ; IETF protocol assignments+ "192.0.0.8"+ "192.0.0.11"+ "192.0.0.255"+ "192.0.2.0" ; TEST-NET-1+ "192.0.2.255"+ "192.88.99.1" ; deprecated 6to4 relay anycast+ "192.88.99.2"+ "198.18.0.0" ; benchmarking+ "198.19.255.255"+ "198.51.100.1" ; TEST-NET-2+ "203.0.113.1" ; TEST-NET-3+ "64:ff9b:1::1" ; local-use IPv4/IPv6 translation+ "64:ff9b::1" ; NAT64 of 0.0.0.1, in "this network"+ "64:ff9b::7f00:1" ; NAT64 of 127.0.0.1+ "64:ff9b::a9fe:a9fe" ; NAT64 of the 169.254.169.254 metadata address+ "100::1" ; discard-only+ "100:0:0:1::1" ; dummy IPv6 prefix+ "2001::1" ; non-global entry inside IETF protocol assignments+ "2001:1::4" ; outside the globally reachable /128 exceptions+ "2001:2::1" ; benchmarking+ "2001:10::1" ; deprecated ORCHID+ "2001:db8::1" ; documentation+ "2002::1" ; 6to4+ "5f00::1" ; segment-routing SIDs+ "1::1" ; reserved IPv6 space outside 2000::/3+ "4000::1"+ "6000::1"+ "8000::1"+ "e000::1"])++(def ^:private reserved-global-unicast-ips+ ;; inside 2000::/3, but above the part IANA has allocated+ ["2d00::1"+ "2e00::1"+ "3000::1"+ "3ffe::1"+ "3fff::1" ; documentation+ "3fff:1000::1" ; just past the documentation block, still reserved+ "3fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"])+(def ^:private non-public-ips- ["127.0.0.1" ; loopback- "169.254.169.254" ; link-local (cloud metadata)- "10.1.2.3" ; RFC1918- "172.16.0.1"- "172.31.255.255"- "192.168.0.1"- "0.0.0.0" ; any-local- "224.0.0.1" ; multicast- "100.64.0.1" ; CGNAT- "100.127.255.255" ; CGNAT (top)- "::1" ; IPv6 loopback- "fe80::1" ; IPv6 link-local- "fc00::1" ; IPv6 ULA (fc)- "fd12:3456::1" ; IPv6 ULA (fd)- "ff02::1" ; IPv6 multicast- "0.1.2.3" ; "this network" 0.0.0.0/8- "0.255.255.255"- "240.0.0.1" ; reserved 240.0.0.0/4- "255.255.255.255" ; limited broadcast (inside 240.0.0.0/4)- "::ffff:127.0.0.1" ; IPv4-mapped loopback- "::ffff:10.0.0.1"]) ; IPv4-mapped RFC1918+ (into ["127.0.0.1" ; loopback+ "169.254.169.254" ; link-local (cloud metadata)+ "10.1.2.3" ; RFC1918+ "172.16.0.1"+ "172.31.255.255"+ "192.168.0.1"+ "0.0.0.0" ; any-local+ "224.0.0.1" ; multicast+ "100.64.0.1" ; CGNAT+ "100.127.255.255" ; CGNAT (top)+ "::1" ; IPv6 loopback+ "fe80::1" ; IPv6 link-local+ "fc00::1" ; IPv6 ULA (fc)+ "fd12:3456::1" ; IPv6 ULA (fd)+ "ff02::1" ; IPv6 multicast+ "0.1.2.3" ; "this network" 0.0.0.0/8+ "0.255.255.255"+ "240.0.0.1" ; reserved 240.0.0.0/4+ "255.255.255.255" ; limited broadcast (inside 240.0.0.0/4)+ "::ffff:127.0.0.1" ; IPv4-mapped loopback+ "::ffff:10.0.0.1"] ; IPv4-mapped RFC1918+ (concat non-global-special-ips reserved-global-unicast-ips)))++(deftest ^:parallel nat64-address-is-judged-by-the-ipv4-it-reaches-test+ (testing (str "a NAT64 gateway translates the low 32 bits of 64:ff9b::/96 to an IPv4 address, so the prefix "+ "being globally reachable says nothing about where a connection to it ends up")+ (are [ip expected] (= expected (http/public-address? (InetAddress/getByName ip)))+ "64:ff9b::808:808" true ; 8.8.8.8+ "64:ff9b::7f00:1" false ; 127.0.0.1+ "64:ff9b::a00:1" false ; 10.0.0.1+ "64:ff9b::a9fe:a9fe" false)) ; 169.254.169.254+ (testing "and allow-private admits the private ones the way it admits the IPv4 they reach"+ (is (true? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName "64:ff9b::a00:1"))))+ (is (false? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName "64:ff9b::7f00:1"))))))(deftest ^:parallel public-address?-test- (testing "globally-routable addresses are allowed"+ (testing "globally reachable addresses are allowed"(doseq [ip public-ips](is (true? (boolean (http/public-address? (InetAddress/getByName ip))))(str "should be public: " ip))))@@ -174,19 +248,60 @@(is (false? (boolean (http/public-address? (InetAddress/getByName ip))))(str "should be rejected: " ip)))))+(defn- proxy-selector+ ^ProxySelector [proxies]+ (proxy [ProxySelector] []+ (select [_uri] proxies)+ (connectFailed [_uri _sa _ioe] nil)))++(deftest ^:parallel jvm-proxied-url?-test+ (testing "a JVM proxy in front of a URL is reported, so a caller knows its :dns-resolver sees only the proxy"+ (binding [http/*proxy-selector* (proxy-selector [(Proxy. Proxy$Type/HTTP (InetSocketAddress. "10.0.0.9" 3128))])]+ (is (true? (http/jvm-proxied-url? "https://api.anthropic.com/v1")))+ (testing "a string that is not a URI is not proxied either"+ (is (false? (http/jvm-proxied-url? "not a url"))))))+ (testing "DIRECT, no selector, and an empty answer all mean unproxied"+ (doseq [proxies [[Proxy/NO_PROXY] []]]+ (binding [http/*proxy-selector* (proxy-selector proxies)]+ (is (false? (http/jvm-proxied-url? "https://api.anthropic.com/v1")))))))++(deftest ^:parallel jvm-proxied-url-route-selection-test+ (let [http-proxy (Proxy. Proxy$Type/HTTP (InetSocketAddress. "10.0.0.9" 3128))+ socks-proxy (Proxy. Proxy$Type/SOCKS (InetSocketAddress. "10.0.0.9" 1080))]+ (doseq [[proxies expected] [[[Proxy/NO_PROXY http-proxy] false]+ [[socks-proxy] false]+ [[socks-proxy Proxy/NO_PROXY http-proxy] false]+ [[http-proxy Proxy/NO_PROXY] true]+ [[socks-proxy http-proxy] true]]]+ (binding [http/*proxy-selector* (proxy-selector proxies)]+ (is (= expected (http/jvm-proxied-url? "https://api.example.com/v1")) (str proxies)))))+ (testing "the selector receives the route target, without a path, query, or user info"+ (let [selected-uri (atom nil)]+ (binding [http/*proxy-selector* (proxy [ProxySelector] []+ (select [uri]+ (reset! selected-uri (str uri))+ [Proxy/NO_PROXY])+ (connectFailed [_uri _sa _ioe] nil))]+ (is (false? (http/jvm-proxied-url? "https://user:[email protected]:8443/v1?key=value")))+ (is (= "https://api.example.com:8443" @selected-uri))))))+(deftest ^:parallel address-allowed-for-network-policy?-test- (testing "external-only admits only globally-routable addresses"+ (testing "external-only admits only globally reachable addresses"(doseq [ip public-ips](is (true? (http/address-allowed-for-network-policy? :external-only (InetAddress/getByName ip))) ip))(doseq [ip non-public-ips](is (false? (http/address-allowed-for-network-policy? :external-only (InetAddress/getByName ip))) ip)))(testing "allow-private re-admits exactly the private ranges -- RFC1918, IPv6 ULA and CGNAT"(doseq [ip ["10.1.2.3" "172.16.0.1" "172.31.255.255" "192.168.0.1"- "100.64.0.1" "100.127.255.255" "fc00::1" "fd12:3456::1"]]+ "100.64.0.1" "100.127.255.255" "fc00::1" "fd12:3456::1"+ "64:ff9b::a00:1"]](is (true? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName ip))) ip)))(testing "allow-private still refuses loopback, link-local, any-local and multicast"(doseq [ip ["127.0.0.1" "::1" "169.254.169.254" "fe80::1" "0.0.0.0" "224.0.0.1" "ff02::1"]](is (false? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName ip))) ip)))+ (testing "allow-private does not admit other non-global special-purpose ranges"+ (doseq [ip non-global-special-ips]+ (is (false? (http/address-allowed-for-network-policy? :allow-private (InetAddress/getByName ip))) ip)))(testing "loopback-and-private admits loopback and the private ranges"(doseq [ip ["127.0.0.1" "127.0.1.5" "::1" "10.1.2.3" "192.168.0.1" "100.64.0.1" "fc00::1"]](is (true? (http/address-allowed-for-network-policy? :loopback-and-private (InetAddress/getByName ip))) ip)))@@ -220,7 +335,16 @@(is (thrown-with-msg? ExceptionInfo #"non-permitted"(.resolve ^DnsResolver (http/network-policy-dns-resolver :external-only) "rebind.example")))(is (= 1 (alength ^"[Ljava.net.InetAddress;"- (.resolve ^DnsResolver (http/network-policy-dns-resolver :allow-private) "rebind.example")))))))+ (.resolve ^DnsResolver (http/network-policy-dns-resolver :allow-private) "rebind.example"))))))+ (testing "the resolver refuses non-global special-purpose addresses under both restricted policies"+ (binding [http/*system-dns-resolver* (doto (InMemoryDnsResolver.)+ (.add "benchmark.example"+ (into-array [(InetAddress/getByName "198.18.0.1")])))]+ (doseq [policy [:external-only :allow-private]]+ (is (thrown-with-msg? ExceptionInfo #"non-permitted"+ (.resolve ^DnsResolver (http/network-policy-dns-resolver policy)+ "benchmark.example"))+ (str policy))))))(deftest ^:parallel fetch-bytes-blocks-without-network-test(testing "blocked URLs return nil at the validation gate, never reaching the network"