The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Allow multiple intermediate ns segments (#82125)
Ben Grabow6780dbbf52e0
diff --git a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj b/.clj-kondo/src/hooks/metabase/settings/models/setting.cljindex 6f00ef88621..cf4f90b3414 100644--- a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj+++ b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj@@ -4,9 +4,9 @@[hooks.common :as common]))(defn lint-defsetting-namespace [node context]- (when-not (re-matches #"^metabase(?:-enterprise)?\.[^\.]+\.settings$" (name (:ns context)))+ (when-not (re-matches #"^metabase(?:-enterprise)?\.([^\.]+\.)+settings$" (name (:ns context)))(hooks/reg-finding! (assoc (meta node)- :message "All defsettings should live in metabase[-enterprise].<module>.settings namespaces"+ :message "All defsettings should live in a namespace ending in .settings under metabase[-enterprise]":type :metabase/defsetting-namespace))));;; TODO -- move this into a Kondo config file in `.clj-kondo/config/`diff --git a/src/metabase/llm/provider.clj b/src/metabase/llm/provider.cljindex ce0d7ba1920..20b73e79491 100644--- a/src/metabase/llm/provider.clj+++ b/src/metabase/llm/provider.clj@@ -18,7 +18,7 @@[[resolve-model-ref]] turns one into the provider type, model, and credentials an adapter needs."(:require[clojure.string :as str]- [metabase.llm.settings :as llm.settings]+ [metabase.llm.provider.settings :as llm.provider.settings][metabase.premium-features.core :as premium-features][metabase.request.current :as request.current][metabase.settings.core :as setting]@@ -38,7 +38,7 @@(def ^:private aws-region-options(mapv (fn [region] {:value region :label region})- (sort llm.settings/known-aws-regions)))+ (sort llm.provider.settings/known-aws-regions)))(def ^:private provider-type-registry"Every provider type Metabase can connect to, in the order the admin UI offers them.@@ -58,7 +58,7 @@:docs-url "https://platform.claude.com/settings/keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -76,7 +76,7 @@:docs-url "https://platform.openai.com/api-keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -94,7 +94,7 @@:docs-url "https://openrouter.ai/keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -112,7 +112,7 @@:docs-url "https://console.mistral.ai/api-keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -130,7 +130,7 @@:docs-url "https://z.ai/manage-apikey/apikey-list"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -147,7 +147,7 @@:docs-url "https://platform.kimi.ai/console/api-keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -167,7 +167,7 @@:docs-url "https://platform.deepseek.com/api_keys"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true@@ -200,7 +200,7 @@:type :text:placeholder (deferred-tru "my-project"):validate (fn [value]- (when-not (llm.settings/valid-google-project-id? value)+ (when-not (llm.provider.settings/valid-google-project-id? value)(tru "\"{0}\" is not a valid Google Cloud project ID. Use the project ID — 6 to 30 lowercase letters, digits and hyphens — rather than the project name or number." value))):help (deferred-tru "The Google Cloud project to use. Optional if the service account key provides it."):docs-url "https://docs.cloud.google.com/resource-manager/docs/creating-managing-projects"}@@ -209,7 +209,7 @@:type :text:placeholder "global":validate (fn [value]- (when-not (llm.settings/valid-google-location? value)+ (when-not (llm.provider.settings/valid-google-location? value)(tru "\"{0}\" is not a valid Google Cloud location." value))):help (deferred-tru "Optional. Defaults to global.")}{:key :auth-method@@ -238,11 +238,11 @@:help (deferred-tru "A short-lived token, e.g. the output of gcloud auth print-access-token. Useful for testing.")}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:advanced? true- :default llm.settings/google-global-api-base-url+ :default llm.provider.settings/google-global-api-base-url:help (deferred-tru "Derived from the location when left at the global host.")}]}{:type "azure":label (deferred-tru "Microsoft Azure")@@ -259,7 +259,7 @@:docs-url "https://ai.azure.com"}{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:required? true@@ -322,7 +322,7 @@:default-model nil:fields [{:key :base-url:normalize strip-trailing-slashes- :validate llm.settings/llm-url-problem+ :validate llm.provider.settings/llm-url-problem:label (deferred-tru "API base URL"):type :text:required? true@@ -396,7 +396,7 @@everything else is always available."[type-name](if (managed-type? type-name)- (some? (llm.settings/llm-proxy-base-url))+ (some? (llm.provider.settings/llm-proxy-base-url))(some? (provider-type type-name))))(defn secret-field-keys@@ -561,7 +561,7 @@proxy is configured."[type-name config](if (managed-type? type-name)- (some? (llm.settings/llm-proxy-base-url))+ (some? (llm.provider.settings/llm-proxy-base-url))(credentials-complete? type-name config)));;; ---------------------------------------- Connections configured by env var ------------------------------------@@ -630,7 +630,7 @@Returns nil for a type no per-provider variable configures — the managed provider, which holds no credentials ofits own, is the only one today. Setting these is the supported way to configure a single connection without writing- JSON into [[metabase.llm.settings/llm-providers]]."+ JSON into [[metabase.llm.provider.settings/llm-providers]]."[type-name](when-let [{:keys [settings]} (get single-provider-settings type-name)](not-empty@@ -689,7 +689,7 @@from [[connections]], so rebuilding the list from there would drop it from the setting the next time an adminsaved anything — the credentials would be gone for good once the env var came back off."[]- (vec (llm.settings/llm-providers)))+ (vec (llm.provider.settings/llm-providers)))(defn- annotated-stored-connections[]@@ -821,7 +821,7 @@(defn set-connections!"Persist `conns` as the stored connection list, dropping the derived annotation keys."[conns]- (llm.settings/set-llm-providers! (mapv #(dissoc % :source :env-vars :env-fields) conns)))+ (llm.provider.settings/set-llm-providers! (mapv #(dissoc % :source :env-vars :env-fields) conns)));;; --------------------------------------------------- Slugs ------------------------------------------------------diff --git a/src/metabase/llm/provider/settings.clj b/src/metabase/llm/provider/settings.cljnew file mode 100644index 00000000000..b0eaa936e42--- /dev/null+++ b/src/metabase/llm/provider/settings.clj@@ -0,0 +1,285 @@+(ns metabase.llm.provider.settings+ "Where LLM provider connections are stored, and the vocabulary the provider type registry validates their+ credential fields against.++ Split out of [[metabase.llm.settings]] so that [[metabase.llm.provider]] can name these directly:+ `metabase.llm.settings` backs its per-provider credential settings with `metabase.llm.provider`, and nothing here+ does."+ (:require+ [clojure.string :as str]+ [metabase.premium-features.core :as premium-features]+ [metabase.request.current :as request.current]+ [metabase.settings.core :as setting :refer [defsetting]]+ [metabase.util :as u]+ [metabase.util.http :as u.http]+ [metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.log :as log])+ (:import+ (java.net MalformedURLException URL)+ (software.amazon.awssdk.regions Region)))++(set! *warn-on-reflection* true)++(def known-aws-regions+ "The set of AWS region ids known to the bundled AWS SDK, e.g. `\"us-east-1\"`.+ Used to validate [[metabase.llm.settings/llm-bedrock-region]]."+ (into #{} (map str) (Region/regions)))++(def ^:private google-project-id-pattern+ "Matches a Google Cloud project ID: 6 to 30 characters of lowercase letters, digits and hyphens, starting with a+ letter and not ending with a hyphen.+ https://docs.cloud.google.com/resource-manager/docs/creating-managing-projects"+ #"[a-z][a-z0-9-]{4,28}[a-z0-9]")++(defn valid-google-project-id?+ "True if `project-id` looks like a valid google project id."+ [project-id]+ (boolean (and (string? project-id)+ (re-matches google-project-id-pattern project-id))))++(def ^:private google-location-pattern+ "Matches a Google Cloud location ID, e.g. `us-central1`: hyphen-separated segments of lowercase letters and digits,+ the first of which starts with a letter."+ #"[a-z][a-z0-9]*(?:-[a-z0-9]+)*")++(def ^:private google-location-max-length+ "The longest location that still leaves a legal DNS label in `{location}-aiplatform.googleapis.com`.+ A label holds 63 characters and the `-aiplatform` suffix takes 11 of them."+ 52)++(defn valid-google-location?+ "True if `location` can be spliced into a Gemini Enterprise Agent Platform request host.+ A location becomes a DNS label of that host, so a value that is not one cannot be sent.+ https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations"+ [location]+ (boolean (and (<= (count location) google-location-max-length)+ (re-matches google-location-pattern location))))++(def google-global-api-base-url+ "Google's global Gemini Enterprise Agent Platform host, and the default+ for [[metabase.llm.settings/llm-google-api-base-url]].+ It serves only the `global` location. A regional location uses `https://{location}-aiplatform.googleapis.com`, and+ the `us` and `eu` multi-region locations use `https://aiplatform.{location}.rep.googleapis.com`."+ "https://aiplatform.googleapis.com")++;;; ------------------------------------------------- Network policy ---------------------------------------------++(def ^:private network-policies+ "The `llm-allowed-networks` policies, loosest last."+ [:external-only :allow-private :allow-all])++(def ^:private network-policy-rank+ (zipmap network-policies (range)))++(defonce ^:private warned-network-policy-values+ (atom #{}))++(defsetting llm-allowed-networks+ (deferred-tru (str "Controls which networks Metabase may connect to for LLM provider base URLs. "+ "Set through the environment only; on Metabase Cloud the default applies.\n"+ "Options:\n"+ "- external-only (default; only globally reachable public addresses)\n"+ "- allow-private (external + private networks but NOT loopback or link-local)\n"+ "- allow-all (no restrictions).\n"+ "The Metabase AI service and LLM proxy are deployment configuration and may always use "+ "private addresses."))+ :type :keyword+ ;; Environment only. A settings manager is who this policy defends against, and on Cloud a customer admin+ ;; loosening it would be reaching for our own infrastructure, so nobody sets it through the API, and a value+ ;; that reached the app DB some other way is ignored rather than trusted.+ :visibility :internal+ :setter :none+ :default :external-only+ :export? false+ :doc (str "Set this when a self-hosted vLLM server is on your private network (allow-private) or on this "+ "machine (allow-all). There is no admin UI for it, and a value stored in the application "+ "database is ignored. With a JVM-wide HTTP(S) proxy, Metabase checks destination addresses "+ "available through local DNS; the deployment proxy must enforce destination restrictions on "+ "its outbound connections. Proxy-only DNS is supported. Metabase enforces destination addresses "+ "at connection time for direct requests.")+ :getter (fn []+ (let [value (some-> (setting/env-var-value :llm-allowed-networks) keyword)]+ (cond+ (nil? value) :external-only+ (contains? network-policy-rank value) value+ ;; fail closed on a typo, and say so once rather than on every request+ :else+ (do (when-not (contains? @warned-network-policy-values value)+ (swap! warned-network-policy-values conj value)+ (log/warnf "Ignoring MB_LLM_ALLOWED_NETWORKS=%s: expected one of %s; using external-only"+ (name value) (str/join ", " (map name network-policies))))+ :external-only)))))++(defn network-policy+ "The network policy for an LLM request.+ `floor`, for a deployment-controlled endpoint such as the AI service, can only loosen [[llm-allowed-networks]]:+ the looser of the two applies."+ ([]+ (llm-allowed-networks))+ ([floor]+ (let [configured (llm-allowed-networks)]+ (if (and floor (> (network-policy-rank floor) (network-policy-rank configured)))+ floor+ configured))))++(defn- host-not-allowed-message+ "Why a base URL on `host` is refused under `policy`, and what to do about it.+ `policy` is the one that actually refused, which a deployment-controlled endpoint's floor may have loosened past+ [[llm-allowed-networks]]: naming a value that is already in force would be advice that changes nothing.+ On Cloud the policy is not the customer's to change."+ [policy host]+ (cond+ (premium-features/is-hosted?)+ (tru "The base URL host {0} is not permitted by Metabase Cloud''s LLM network policy. Use an LLM provider on the public internet." host)++ (= :allow-private policy)+ (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-all for a server on this machine." host)++ :else+ (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-private for a server on your private network, or allow-all for one on this machine." host)))++(defn- url-not-allowed-ex+ "The 400 every policy refusal is thrown as, at set time and at connection time alike.+ `:status` sits beside `:status-code` because the semantic-search dead-letter queue files errors by `:status`, and a+ refused endpoint is a permanent failure, not one to retry on the fast schedule."+ ([message host]+ (url-not-allowed-ex message host nil))+ ([message host cause]+ (ex-info message+ {:status-code 400+ :status 400+ :api-error true+ :error-code :llm-host-not-allowed+ :llm-host host}+ cause)))++(defn llm-url-syntax-problem+ "Why `url` cannot be an LLM provider base URL under any policy, or nil when it can: it must be an `http` or+ `https` URL that names a host and carries no username or password.+ Nothing here resolves the host, so it is cheap enough to run on every request.+ A blank `url` is not a problem here: the not-configured handling covers it."+ [url]+ (when-not (str/blank? url)+ (let [^URL parsed (try+ (URL. ^String url)+ (catch MalformedURLException _ nil))]+ (cond+ (not (and parsed (#{"http" "https"} (.getProtocol parsed)) (not-empty (.getHost parsed))))+ (tru "Invalid base URL: it must start with http:// or https://.")++ ;; it would otherwise ride along into error messages and ex-data+ (some? (.getUserInfo parsed))+ (tru "Invalid base URL: it must not contain a username or password.")))))++(defn llm-url-problem+ "Why `url` may not be used as an LLM provider base URL, or nil when it may: [[llm-url-syntax-problem]], and every+ address the host resolves to must be permitted by the network policy. The one-argument form uses+ [[llm-allowed-networks]].+ Used on write and before proxied requests. An unresolved host is permitted to allow proxy-only DNS.+ Direct requests use the policy resolver from [[llm-request-opts]] to enforce the policy at connection time."+ ([url]+ (llm-url-problem (llm-allowed-networks) url))+ ([network-policy url]+ (or (llm-url-syntax-problem url)+ (when-not (str/blank? url)+ (let [host (.getHost (URL. ^String url))]+ (when-not (u.http/host-allowed-for-network-policy? network-policy host)+ (host-not-allowed-message network-policy host)))))))++(defn llm-request-opts+ "clj-http options that put the network policy on a request to `url`: redirects are never followed, and a+ `:dns-resolver` refuses any address the policy does not permit (omitted under `:allow-all`). Throws the same 400 as+ a set-time refusal when `url` fails [[llm-url-syntax-problem]]. `floor` is for a deployment-controlled endpoint,+ see [[network-policy]]. Pair with [[rethrow-if-llm-network-policy-error!]] around the request.++ Behind a JVM-wide proxy a `:dns-resolver` enforces nothing, so the check moves to [[llm-url-problem]] here; see+ [[metabase.util.http/jvm-proxied-url?]]."+ ([url]+ (llm-request-opts nil url))+ ([floor url]+ (when-let [problem (llm-url-syntax-problem url)]+ (throw (url-not-allowed-ex problem (u.http/->hostname url))))+ (let [policy (network-policy floor)]+ (if (u.http/jvm-proxied-url? url)+ ;; An operator who configured a JVM proxy put it between Metabase and everything, deliberately, so it is+ ;; trusted rather than judged by the policy -- which is also what makes a private egress proxy usable under+ ;; `:external-only`. The proxy resolves the target on its own, so the target is checked here instead. That+ ;; leaves proxy-only DNS, differing DNS answers, and rebinding to the proxy's own destination policy;+ ;; Metabase cannot enforce the addresses the proxy connects to.+ (do (when-let [problem (llm-url-problem policy url)]+ (throw (url-not-allowed-ex problem (u.http/->hostname url))))+ {:redirect-strategy :none})+ ;; nil under :allow-all, which leaves clj-http on its default resolver. Redirects stay disabled under every+ ;; policy: the resolver would stop an internal target, but credentials could otherwise follow a redirect to a+ ;; public host.+ (u/assoc-dissoc {:redirect-strategy :none}+ :dns-resolver (u.http/network-policy-dns-resolver policy))))))++(defn llm-network-policy-error?+ "Whether `e` or one of its causes is a connection-time refusal from the policy DNS resolver."+ [e]+ (boolean (:ssrf (u/all-ex-data e))))++(defn rethrow-if-llm-network-policy-error!+ "Translate a connection-time refusal from the policy DNS resolver in `e` to the 400 a set-time refusal gets.+ Returns nil when `e` is unrelated. Logs the refusal: it is the only trace a host that rebinds leaves.+ The resolver records the policy it refused under, which is what the message and the log line report."+ [e url]+ (when (llm-network-policy-error? e)+ (let [host (u.http/->hostname url)+ policy (or (:policy (u/all-ex-data e)) (llm-allowed-networks))]+ (log/warnf "Refused an LLM request to %s: it resolves to an address the %s network policy does not permit"+ host (name policy))+ (throw (url-not-allowed-ex (host-not-allowed-message policy host) host e)))))++;;; ---------------------------------------------- Provider connections ------------------------------------------++(def ^:dynamic *allow-llm-provider-write*+ "Whether a trusted provider API operation may persist [[llm-providers]] during an HTTP request."+ false)++(defsetting llm-providers+ (deferred-tru "JSON array of configured LLM provider connections. Each entry has a `key` (a URL-safe slug identifying the connection), a `type` (the provider type, e.g. `anthropic`), a display `name`, and a `config` map of that provider type''s credential fields.")+ :type :json+ :default []+ :encryption :when-encryption-key-set+ :sensitive? true+ :visibility :internal+ :export? false+ :audit :no-value+ :setter (fn [new-value]+ ;; Startup configuration and backend callers have no current request. During one, only the dedicated+ ;; provider API may write the backing setting; the generic settings API cannot perform its validation+ ;; or prove that secrets accompanying a base-URL change were freshly supplied.+ (when (and (request.current/current-request)+ (not *allow-llm-provider-write*))+ (throw (ex-info (tru "Manage LLM provider connections through the provider connection settings.")+ {:status-code 400+ :api-error true+ :error-code :llm-providers-direct-write-forbidden})))+ ((requiring-resolve 'metabase.llm.provider/validate-changed-connections!) new-value)+ (setting/set-value-of-type! :json :llm-providers new-value))+ :doc "Connections are normally managed from the admin AI settings page. Setting this environment variable puts the whole list under environment control and makes it read-only in the UI.++Configuring a provider through the single-provider variables (`MB_LLM_ANTHROPIC_API_KEY` and friends) is equally supported, and is the simpler option when you only need one connection per provider and would rather not hand-write JSON. Each such provider becomes a read-only connection whose key is the provider type, resolved from the environment on every read, so editing one of those variables is picked up on the next restart. A provider configured this way takes precedence over a stored connection with the same key.")++(defn set-llm-providers!+ "Allow the provider API and testing-only fixture endpoint to persist connections. Other HTTP paths must not write+ the backing setting directly."+ [providers]+ (binding [*allow-llm-provider-write* true]+ (llm-providers! providers)))++;;; --------------------------------------------------- Proxy ---------------------------------------------------++(defsetting llm-proxy-base-url+ (deferred-tru "Base URL for the LLM proxy. When set, requests to the managed Metabase AI service are routed through this proxy and authenticated with the instance token instead of a provider API key. Harbormaster adds /llm component into the url.")+ ;; For details on llm component see the https://github.com/metabase/metabase/pull/74526#discussion_r3282553435.+ :enabled? #(or (premium-features/has-feature? :metabase-ai-managed)+ (premium-features/has-feature? :offer-metabase-ai-managed)+ (premium-features/has-feature? :metabot-v3))+ :encryption :when-encryption-key-set+ :visibility :internal+ :default nil+ :export? false+ :doc false)diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.cljindex 809d1a50f03..929697925fc 100644--- a/src/metabase/llm/settings.clj+++ b/src/metabase/llm/settings.clj@@ -3,23 +3,37 @@(:require[clojure.string :as str][metabase.config.core :as config]+ [metabase.llm.provider :as llm.provider]+ [metabase.llm.provider.settings][metabase.premium-features.core :as premium-features]- [metabase.request.current :as request.current]- [metabase.settings.core :as setting :refer [defsetting]]+ [metabase.settings.core :refer [defsetting]][metabase.util :as u]- [metabase.util.http :as u.http][metabase.util.i18n :refer [deferred-tru tru]]- [metabase.util.log :as log])+ [potemkin :as p])(:import- (java.net MalformedURLException URL)- (software.amazon.awssdk.regions Region)))+ (java.net MalformedURLException URL)))(set! *warn-on-reflection* true)-(def known-aws-regions- "The set of AWS region ids known to the bundled AWS SDK, e.g. `\"us-east-1\"`.- Used to validate [[llm-bedrock-region]]."- (into #{} (map str) (Region/regions)))+;; kept on this namespace so callers are unaffected by the split+(p/import-vars+ [metabase.llm.provider.settings+ google-global-api-base-url+ known-aws-regions+ llm-allowed-networks+ llm-network-policy-error?+ llm-providers+ llm-providers!+ llm-proxy-base-url+ llm-proxy-base-url!+ llm-request-opts+ llm-url-problem+ llm-url-syntax-problem+ network-policy+ rethrow-if-llm-network-policy-error!+ set-llm-providers!+ valid-google-location?+ valid-google-project-id?])(def ^:private loopback-hosts"Hostnames that resolve to the local machine. `URL.getHost` returns IPv6 hosts@@ -44,189 +58,20 @@{:status-code 400:llm-url url}))))))-(def ^:private network-policies- "The `llm-allowed-networks` policies, loosest last."- [:external-only :allow-private :allow-all])--(def ^:private network-policy-rank- (zipmap network-policies (range)))--(defonce ^:private warned-network-policy-values- (atom #{}))--(defsetting llm-allowed-networks- (deferred-tru (str "Controls which networks Metabase may connect to for LLM provider base URLs. "- "Set through the environment only; on Metabase Cloud the default applies.\n"- "Options:\n"- "- external-only (default; only globally reachable public addresses)\n"- "- allow-private (external + private networks but NOT loopback or link-local)\n"- "- allow-all (no restrictions).\n"- "The Metabase AI service and LLM proxy are deployment configuration and may always use "- "private addresses."))- :type :keyword- ;; Environment only. A settings manager is who this policy defends against, and on Cloud a customer admin- ;; loosening it would be reaching for our own infrastructure, so nobody sets it through the API, and a value- ;; that reached the app DB some other way is ignored rather than trusted.- :visibility :internal- :setter :none- :default :external-only- :export? false- :doc (str "Set this when a self-hosted vLLM server is on your private network (allow-private) or on this "- "machine (allow-all). There is no admin UI for it, and a value stored in the application "- "database is ignored. With a JVM-wide HTTP(S) proxy, Metabase checks destination addresses "- "available through local DNS; the deployment proxy must enforce destination restrictions on "- "its outbound connections. Proxy-only DNS is supported. Metabase enforces destination addresses "- "at connection time for direct requests.")- :getter (fn []- (let [value (some-> (setting/env-var-value :llm-allowed-networks) keyword)]- (cond- (nil? value) :external-only- (contains? network-policy-rank value) value- ;; fail closed on a typo, and say so once rather than on every request- :else- (do (when-not (contains? @warned-network-policy-values value)- (swap! warned-network-policy-values conj value)- (log/warnf "Ignoring MB_LLM_ALLOWED_NETWORKS=%s: expected one of %s; using external-only"- (name value) (str/join ", " (map name network-policies))))- :external-only)))))--(defn network-policy- "The network policy for an LLM request.- `floor`, for a deployment-controlled endpoint such as the AI service, can only loosen [[llm-allowed-networks]]:- the looser of the two applies."- ([]- (llm-allowed-networks))- ([floor]- (let [configured (llm-allowed-networks)]- (if (and floor (> (network-policy-rank floor) (network-policy-rank configured)))- floor- configured))))--(defn- host-not-allowed-message- "Why a base URL on `host` is refused under `policy`, and what to do about it.- `policy` is the one that actually refused, which a deployment-controlled endpoint's floor may have loosened past- [[llm-allowed-networks]]: naming a value that is already in force would be advice that changes nothing.- On Cloud the policy is not the customer's to change."- [policy host]- (cond- (premium-features/is-hosted?)- (tru "The base URL host {0} is not permitted by Metabase Cloud''s LLM network policy. Use an LLM provider on the public internet." host)-- (= :allow-private policy)- (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-all for a server on this machine." host)-- :else- (tru "The base URL host {0} is on a network Metabase is not allowed to connect to. Set MB_LLM_ALLOWED_NETWORKS=allow-private for a server on your private network, or allow-all for one on this machine." host)))--(defn- url-not-allowed-ex- "The 400 every policy refusal is thrown as, at set time and at connection time alike.- `:status` sits beside `:status-code` because the semantic-search dead-letter queue files errors by `:status`, and a- refused endpoint is a permanent failure, not one to retry on the fast schedule."- ([message host]- (url-not-allowed-ex message host nil))- ([message host cause]- (ex-info message- {:status-code 400- :status 400- :api-error true- :error-code :llm-host-not-allowed- :llm-host host}- cause)))--(defn llm-url-syntax-problem- "Why `url` cannot be an LLM provider base URL under any policy, or nil when it can: it must be an `http` or- `https` URL that names a host and carries no username or password.- Nothing here resolves the host, so it is cheap enough to run on every request.- A blank `url` is not a problem here: the not-configured handling covers it."- [url]- (when-not (str/blank? url)- (let [^URL parsed (try- (URL. ^String url)- (catch MalformedURLException _ nil))]- (cond- (not (and parsed (#{"http" "https"} (.getProtocol parsed)) (not-empty (.getHost parsed))))- (tru "Invalid base URL: it must start with http:// or https://.")-- ;; it would otherwise ride along into error messages and ex-data- (some? (.getUserInfo parsed))- (tru "Invalid base URL: it must not contain a username or password.")))))--(defn llm-url-problem- "Why `url` may not be used as an LLM provider base URL, or nil when it may: [[llm-url-syntax-problem]], and every- address the host resolves to must be permitted by the network policy. The one-argument form uses- [[llm-allowed-networks]].- Used on write and before proxied requests. An unresolved host is permitted to allow proxy-only DNS.- Direct requests use the policy resolver from [[llm-request-opts]] to enforce the policy at connection time."- ([url]- (llm-url-problem (llm-allowed-networks) url))- ([network-policy url]- (or (llm-url-syntax-problem url)- (when-not (str/blank? url)- (let [host (.getHost (URL. ^String url))]- (when-not (u.http/host-allowed-for-network-policy? network-policy host)- (host-not-allowed-message network-policy host)))))))--(defn llm-request-opts- "clj-http options that put the network policy on a request to `url`: redirects are never followed, and a- `:dns-resolver` refuses any address the policy does not permit (omitted under `:allow-all`). Throws the same 400 as- a set-time refusal when `url` fails [[llm-url-syntax-problem]]. `floor` is for a deployment-controlled endpoint,- see [[network-policy]]. Pair with [[rethrow-if-llm-network-policy-error!]] around the request.-- Behind a JVM-wide proxy a `:dns-resolver` enforces nothing, so the check moves to [[llm-url-problem]] here; see- [[metabase.util.http/jvm-proxied-url?]]."- ([url]- (llm-request-opts nil url))- ([floor url]- (when-let [problem (llm-url-syntax-problem url)]- (throw (url-not-allowed-ex problem (u.http/->hostname url))))- (let [policy (network-policy floor)]- (if (u.http/jvm-proxied-url? url)- ;; An operator who configured a JVM proxy put it between Metabase and everything, deliberately, so it is- ;; trusted rather than judged by the policy -- which is also what makes a private egress proxy usable under- ;; `:external-only`. The proxy resolves the target on its own, so the target is checked here instead. That- ;; leaves proxy-only DNS, differing DNS answers, and rebinding to the proxy's own destination policy;- ;; Metabase cannot enforce the addresses the proxy connects to.- (do (when-let [problem (llm-url-problem policy url)]- (throw (url-not-allowed-ex problem (u.http/->hostname url))))- {:redirect-strategy :none})- ;; nil under :allow-all, which leaves clj-http on its default resolver. Redirects stay disabled under every- ;; policy: the resolver would stop an internal target, but credentials could otherwise follow a redirect to a- ;; public host.- (u/assoc-dissoc {:redirect-strategy :none}- :dns-resolver (u.http/network-policy-dns-resolver policy))))))--(defn llm-network-policy-error?- "Whether `e` or one of its causes is a connection-time refusal from the policy DNS resolver."- [e]- (boolean (:ssrf (u/all-ex-data e))))--(defn rethrow-if-llm-network-policy-error!- "Translate a connection-time refusal from the policy DNS resolver in `e` to the 400 a set-time refusal gets.- Returns nil when `e` is unrelated. Logs the refusal: it is the only trace a host that rebinds leaves.- The resolver records the policy it refused under, which is what the message and the log line report."- [e url]- (when (llm-network-policy-error? e)- (let [host (u.http/->hostname url)- policy (or (:policy (u/all-ex-data e)) (llm-allowed-networks))]- (log/warnf "Refused an LLM request to %s: it resolves to an address the %s network policy does not permit"- host (name policy))- (throw (url-not-allowed-ex (host-not-allowed-message policy host) host e)))))-;; TODO (Chris 2026-08-17) -- BOT-2005: generate-sql and semantic search read these settings directly, so;; deleting the connection they key off turns those features off. They should name a connection instead.(defn- connection-field-getter- "Getter for a per-provider credential setting whose value lives on the `llm-providers` connection list.- Resolved late: [[metabase.llm.provider]] requires this namespace."+ "Getter for a per-provider credential setting whose value lives on the `llm-providers` connection list."[setting-kw](fn []- ((requiring-resolve 'metabase.llm.provider/single-provider-setting-value) setting-kw)))+ (llm.provider/single-provider-setting-value setting-kw)))(defn- connection-field-setter"Setter counterpart of [[connection-field-getter]], writing through to the connection list so `config.yml`provisioning and code that has always written these settings keep working."[setting-kw](fn [new-value]- ((requiring-resolve 'metabase.llm.provider/set-single-provider-setting!) setting-kw new-value)))+ (llm.provider/set-single-provider-setting! setting-kw new-value)));;; ------------------------------------------------- Anthropic -------------------------------------------------@@ -437,36 +282,6 @@:setter (connection-field-setter :llm-google-oauth-access-token):doc "Backed by the google connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")-(def ^:private google-project-id-pattern- "Matches a Google Cloud project ID: 6 to 30 characters of lowercase letters, digits and hyphens, starting with a- letter and not ending with a hyphen.- https://docs.cloud.google.com/resource-manager/docs/creating-managing-projects"- #"[a-z][a-z0-9-]{4,28}[a-z0-9]")--(defn valid-google-project-id?- "True if `project-id` looks like a valid google project id."- [project-id]- (boolean (and (string? project-id)- (re-matches google-project-id-pattern project-id))))--(def ^:private google-location-pattern- "Matches a Google Cloud location ID, e.g. `us-central1`: hyphen-separated segments of lowercase letters and digits,- the first of which starts with a letter."- #"[a-z][a-z0-9]*(?:-[a-z0-9]+)*")--(def ^:private google-location-max-length- "The longest location that still leaves a legal DNS label in `{location}-aiplatform.googleapis.com`.- A label holds 63 characters and the `-aiplatform` suffix takes 11 of them."- 52)--(defn valid-google-location?- "True if `location` can be spliced into a Gemini Enterprise Agent Platform request host.- A location becomes a DNS label of that host, so a value that is not one cannot be sent.- https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations"- [location]- (boolean (and (<= (count location) google-location-max-length)- (re-matches google-location-pattern location))))-(defsetting llm-google-project-id(deferred-tru "The Google Cloud project ID for the Gemini Enterprise Agent Platform."):encryption :no@@ -485,12 +300,6 @@:setter (connection-field-setter :llm-google-location):doc "Backed by the google connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")-(def google-global-api-base-url- "Google's global Gemini Enterprise Agent Platform host, and the default for [[llm-google-api-base-url]].- It serves only the `global` location. A regional location uses `https://{location}-aiplatform.googleapis.com`, and- the `us` and `eu` multi-region locations use `https://aiplatform.{location}.rep.googleapis.com`."- "https://aiplatform.googleapis.com")-(defsetting llm-google-api-base-url(deferred-tru "The Gemini Enterprise Agent Platform API base URL. Leave unset to derive it from the location."):encryption :when-encryption-key-set@@ -607,63 +416,13 @@:visibility :settings-manager:export? false)-;;; ---------------------------------------------- Provider connections -------------------------------------------;;; The per-provider credential settings above are read-only at runtime: they configure a connection only when set;;; by an environment variable, which [[metabase.llm.provider/connections]] resolves on every read. Editing one in;;; the app DB would not reach the connection serving requests, so a write is rejected rather than silently ignored.;;; Connections are managed through the `/api/llm/providers` endpoints instead.-(def ^:dynamic *allow-llm-provider-write*- "Whether a trusted provider API operation may persist [[llm-providers]] during an HTTP request."- false)--(defsetting llm-providers- (deferred-tru "JSON array of configured LLM provider connections. Each entry has a `key` (a URL-safe slug identifying the connection), a `type` (the provider type, e.g. `anthropic`), a display `name`, and a `config` map of that provider type''s credential fields.")- :type :json- :default []- :encryption :when-encryption-key-set- :sensitive? true- :visibility :internal- :export? false- :audit :no-value- :setter (fn [new-value]- ;; Startup configuration and backend callers have no current request. During one, only the dedicated- ;; provider API may write the backing setting; the generic settings API cannot perform its validation- ;; or prove that secrets accompanying a base-URL change were freshly supplied.- (when (and (request.current/current-request)- (not *allow-llm-provider-write*))- (throw (ex-info (tru "Manage LLM provider connections through the provider connection settings.")- {:status-code 400- :api-error true- :error-code :llm-providers-direct-write-forbidden})))- ((requiring-resolve 'metabase.llm.provider/validate-changed-connections!) new-value)- (setting/set-value-of-type! :json :llm-providers new-value))- :doc "Connections are normally managed from the admin AI settings page. Setting this environment variable puts the whole list under environment control and makes it read-only in the UI.--Configuring a provider through the single-provider variables (`MB_LLM_ANTHROPIC_API_KEY` and friends) is equally supported, and is the simpler option when you only need one connection per provider and would rather not hand-write JSON. Each such provider becomes a read-only connection whose key is the provider type, resolved from the environment on every read, so editing one of those variables is picked up on the next restart. A provider configured this way takes precedence over a stored connection with the same key.")--(defn set-llm-providers!- "Allow the provider API and testing-only fixture endpoint to persist connections. Other HTTP paths must not write- the backing setting directly."- [providers]- (binding [*allow-llm-provider-write* true]- (llm-providers! providers)))-;;; --------------------------------------------------- Proxy ----------------------------------------------------(defsetting llm-proxy-base-url- (deferred-tru "Base URL for the LLM proxy. When set, requests to the managed Metabase AI service are routed through this proxy and authenticated with the instance token instead of a provider API key. Harbormaster adds /llm component into the url.")- ;; For details on llm component see the https://github.com/metabase/metabase/pull/74526#discussion_r3282553435.- :enabled? #(or (premium-features/has-feature? :metabase-ai-managed)- (premium-features/has-feature? :offer-metabase-ai-managed)- (premium-features/has-feature? :metabot-v3))- :encryption :when-encryption-key-set- :visibility :internal- :default nil- :export? false- :doc false)-(defsetting ai-service-base-url(deferred-tru "Base URL for the managed Metabase AI service."):enabled? #(or (premium-features/has-feature? :metabase-ai-managed)diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex 84bb5124031..fd747fba44a 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -273,13 +273,16 @@"True when changing `setting-key` could change whether Metabot can reach an LLM — i.e. itfeeds [[llm-metabot-configured?]] or one of the Metabot enable settings.- Matches all of [[metabase.llm.settings]] rather than a hand-listed key set: being broad- costs a redundant re-check, while missing a key silently strands callers that wake on it."+ Matches every setting the `llm` module defines, rather than a hand-listed key set or a single+ namespace: the module spreads its settings over several namespaces, and being broad costs a+ redundant re-check while missing a key silently strands callers that wake on it."[setting-key](boolean(or (contains? metabot-llm-setting-keys setting-key)- (= 'metabase.llm.settings- (:namespace (get @setting/registered-settings setting-key))))))+ (some-> (get @setting/registered-settings setting-key)+ :namespace+ str+ (str/starts-with? "metabase.llm.")))));;; ------------------------------------------------- AI Data Retention ------------------------------------------------diff --git a/src/metabase/util/experiment/settings.clj b/src/metabase/util/experiment/settings.cljindex fe00f6be6ba..6176fe1f62d 100644--- a/src/metabase/util/experiment/settings.clj+++ b/src/metabase/util/experiment/settings.clj@@ -4,8 +4,6 @@[metabase.util.experiment :as experiment][metabase.util.i18n :refer [deferred-tru]]))-;; experiment lives under util rather than a top-level module, so no <module>.settings ns exists for it-#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}(defsetting experiments-enabled(deferred-tru "Enable or disable all code experiments. When disabled, only the production code path runs."):type :boolean