The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
metabot: ability to make requests via ai proxy w/o having api key configured (#71714)
Oleksandr Solovyov3671d3bd8007
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex bfbb7b05e0d..7fb63e8c868 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1007,7 +1007,8 @@:api #{metabase.internal-stats.coremetabase.internal-stats.metabot}:uses #{app-db- models}+ models+ util}:model-imports #{:model/Card:model/Dashboard:model/MetabotMessage@@ -1100,8 +1101,7 @@llm{:team "Metabot"- :api #{metabase.llm.anthropic- metabase.llm.api+ :api #{metabase.llm.apimetabase.llm.initmetabase.llm.settings}:uses #{analytics@@ -1109,6 +1109,7 @@driverliblib-be+ metabotmodelsrequestsettings@@ -1179,7 +1180,10 @@{:team "Metabot":api #{metabase.metabot.apimetabase.metabot.api.entity-analysis- metabase.metabot.init}+ metabase.metabot.core+ metabase.metabot.init+ metabase.metabot.self+ metabase.metabot.settings}:friends #{agent-apienterprise/metabotslackbot}diff --git a/resources/migrations/060/20260401_metabot_message_ai_proxied.yaml b/resources/migrations/060/20260401_metabot_message_ai_proxied.yamlnew file mode 100644index 00000000000..d71aa501dc7--- /dev/null+++ b/resources/migrations/060/20260401_metabot_message_ai_proxied.yaml@@ -0,0 +1,18 @@+## Track whether metabot messages were routed through the AI proxy++databaseChangeLog:+ - logicalFilePath: migrations/060_update_migrations.yaml+ - changeSet:+ id: v60.2026-04-01T15:00:00+ author: solovyov+ comment: Add ai_proxied flag to metabot_message to track proxy vs BYOK routing+ changes:+ - addColumn:+ tableName: metabot_message+ columns:+ - column:+ name: ai_proxied+ type: ${boolean.type}+ remarks: Whether this message was routed through the Metabase Cloud AI proxy (true) or directly via BYOK keys (false). Null for messages created before this migration.+ constraints:+ nullable: truediff --git a/src/metabase/internal_stats/metabot.clj b/src/metabase/internal_stats/metabot.cljindex b866188481a..2ef2e6f76a0 100644--- a/src/metabase/internal_stats/metabot.clj+++ b/src/metabase/internal_stats/metabot.clj@@ -1,25 +1,48 @@(ns metabase.internal-stats.metabot(:require+ [clojure.string :as str][java-time.api :as t]+ [metabase.util :as u][toucan2.core :as t2]))(defn metabot-stats"Calculate total Metabot token usage over a window of the previous UTC day 00:00-23:59"[](let [yesterday-utc (-> (t/offset-date-time (t/zone-offset "+00"))- (t/minus (t/days 1)))]- {:metabot-tokens (or (t2/select-one-fn :sum- [:model/MetabotMessage [:%sum.total_tokens :sum]]- {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})- 0)- :metabot-queries (t2/select-one-fn :cnt- [:model/MetabotMessage [:%count.id :cnt]]- ;; there are requests from users and responses from ai-service, in theory- ;; counting requests from users should be good enough- :role "user"- {:where [:= [:cast :created_at :date] [:cast yesterday-utc :date]]})- :metabot-users (:cnt (t2/query-one {:select [[[:count [:distinct :c.user_id]] :cnt]]- :from [[:metabot_message :m]]- :join [[:metabot_conversation :c] [:= :c.id :m.conversation_id]]- :where [:= [:cast :m.created_at :date] [:cast yesterday-utc :date]]}))- :metabot-usage-date (str (t/local-date yesterday-utc))}))+ (t/minus (t/days 1)))+ tokens (or (t2/select-one-fn :sum+ [:model/MetabotMessage [:%sum.total_tokens :sum]]+ {:where [:and+ :ai_proxied+ [:= [:cast :created_at :date] [:cast yesterday-utc :date]]+ [:not= :usage nil]]})+ 0)]+ (when (pos? tokens)+ {:metabot-tokens (long tokens)+ :metabot-usage (let [usages (t2/select-fn-vec :usage+ [:model/MetabotMessage :usage]+ {:where [:and+ :ai_proxied+ [:= [:cast :created_at :date] [:cast yesterday-utc :date]]+ [:not= :usage nil]]})]+ (->> (for [usage usages+ [prov-model tokens] usage+ :let [k (str/replace-first (u/qualified-name prov-model) "/" ":")]]+ {(str k ":in") (:prompt tokens)+ (str k ":out") (:completion tokens)})+ (apply merge-with +)))+ :metabot-queries (t2/select-one-fn :cnt+ [:model/MetabotMessage [:%count.id :cnt]]+ ;; there are requests from users and responses from ai-service, in theory+ ;; counting requests from users should be good enough+ :role "user"+ {:where [:and+ :ai_proxied+ [:= [:cast :created_at :date] [:cast yesterday-utc :date]]]})+ :metabot-users (:cnt (t2/query-one {:select [[[:count [:distinct :c.user_id]] :cnt]]+ :from [[:metabot_message :m]]+ :join [[:metabot_conversation :c] [:= :c.id :m.conversation_id]]+ :where [:and+ :ai_proxied+ [:= [:cast :m.created_at :date] [:cast yesterday-utc :date]]]}))+ :metabot-usage-date (str (t/local-date yesterday-utc))})))diff --git a/src/metabase/llm/anthropic.clj b/src/metabase/llm/anthropic.cljindex 0537ce75866..f7fdc4fdd3b 100644--- a/src/metabase/llm/anthropic.clj+++ b/src/metabase/llm/anthropic.clj@@ -5,7 +5,6 @@using tool_use for structured output."(:require[clj-http.client :as http]- [clojure.core.memoize :as memoize][clojure.string :as str][metabase.llm.settings :as llm.settings][metabase.util :as u]@@ -79,37 +78,6 @@{:type :llm-not-configured})))api-key))-(def ^:private list-models*- "Memoized implementation of list-models with 5-minute TTL."- (memoize/ttl- (fn [{:keys [api-key api-url api-version]}]- (try- (let [url (str api-url "/v1/models")- response (http/get url- {:headers {"x-api-key" api-key- "anthropic-version" api-version}})- body (json/decode+kw (:body response))- models (reverse (sort-by :created_at (:data body)))]- {:models (map #(select-keys % [:id :display_name]) models)})- (catch Exception e- (handle-api-error e))))- :ttl/threshold (* 5 60 1000)))--(defn list-models- "Send a list models request to Anthropic.- Returns a map with :models. Results are cached for 5 minutes."- ([]- (list-models {:api-key (get-api-key-or-throw)}))- ([{:keys [api-key api-url api-version]- :or {api-url (llm.settings/llm-anthropic-api-base-url)- api-version (llm.settings/llm-anthropic-api-version)}}]- (when (str/blank? api-key)- (throw (ex-info "LLM is not configured. Please set an Anthropic API key via MB_LLM_ANTHROPIC_API_KEY."- {:type :llm-not-configured})))- (list-models* {:api-key api-key- :api-url api-url- :api-version api-version})))-(defn chat-completion"Send a chat completion request to Anthropic.Returns a map with:diff --git a/src/metabase/llm/api.clj b/src/metabase/llm/api.cljindex fa8d9836eb8..b9d18c5f9b3 100644--- a/src/metabase/llm/api.clj+++ b/src/metabase/llm/api.clj@@ -13,6 +13,9 @@[metabase.llm.anthropic :as llm.anthropic][metabase.llm.context :as llm.context][metabase.llm.settings :as llm.settings]+ [metabase.metabot.core :as metabot]+ [metabase.metabot.self :as metabot.self]+ [metabase.metabot.settings :as metabot.settings][metabase.request.core :as request][metabase.util :as u][metabase.util.i18n :refer [tru]]@@ -90,13 +93,16 @@[:display_name :string]]]]]"List available LLM models from the configured provider.- Requires LLM to be configured (Anthropic API key set in admin settings)."+ Requires LLM to be configured for the selected provider in admin settings."[_route-params_query-params]- (when-not (llm.settings/llm-anthropic-api-key)- (throw (ex-info (tru "LLM is not configured. Please set an Anthropic API key in admin settings.")+ (when-not (metabot.settings/llm-metabot-configured?)+ (throw (ex-info (tru "LLM is not configured. Please configure the selected provider in admin settings."){:status-code 403})))- (llm.anthropic/list-models))+ (let [provider-and-model (metabot.settings/llm-metabot-provider)+ ai-proxy? (metabot/metabase-provider? provider-and-model)+ provider (metabot/provider-and-model->provider provider-and-model)]+ (metabot.self/list-models provider {:ai-proxy? ai-proxy?})))(def ^:private table-with-columns-schema"Schema for table metadata with columns returned by /extract-tables."diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.cljindex 9538e13ae17..78d4100122d 100644--- a/src/metabase/llm/settings.clj+++ b/src/metabase/llm/settings.clj@@ -120,6 +120,17 @@(deferred-tru "Invalid OpenRouter API key format. Key must start with ''sk-or-v1-''.")):doc false)+;;; --------------------------------------------------- Proxy ---------------------------------------------------++(defsetting llm-proxy-base-url+ (deferred-tru "Base URL for the LLM proxy. When set, all LLM requests are routed through this proxy and authenticated with the instance token instead of provider API keys.")+ :feature :metabase-ai-provider+ :encryption :no+ :visibility :internal+ :default nil+ :export? false+ :doc false)+;;; -------------------------------------------------- General --------------------------------------------------(defsetting llm-max-tokensdiff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 06c4c8dca76..2dd096e4818 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -13,6 +13,7 @@[metabase.metabot.agent.messages :as messages][metabase.metabot.agent.profiles :as profiles][metabase.metabot.agent.streaming :as streaming]+ [metabase.metabot.provider-util :as provider-util][metabase.metabot.scope :as scope][metabase.metabot.self :as self][metabase.metabot.settings :as metabot.settings]@@ -444,16 +445,25 @@(defn- accumulate-usage-xf"Transducer that merges each `:usage` part into the cumulative usage atom- (keyed by model) and replaces the part's `:usage` with the running total.+ (keyed by provider-and-model) and replaces the part's `:usage` with the running total.+ Also sets `:model` on the part to `provider-and-model` so downstream consumers+ (e.g. `extract-usage`) key usage by the canonical provider/model string rather+ than the raw model name returned by the API.++ The `metabase/` routing prefix is stripped so usage keys reflect the actual+ provider/model (e.g. `openrouter/anthropic/claude-haiku-4-5`) regardless of+ whether the request was routed through the AI proxy.Non-usage parts pass through unchanged."- [usage-atom]- (map (fn [{:keys [usage model] :as part}]- (if (= (:type part) :usage)- (let [model (or model "unknown")]- (assoc part :usage- (-> (swap! usage-atom update model (partial merge-with +) usage)- (get model))))- part))))+ [usage-atom provider-and-model]+ (let [model (or (some-> provider-and-model provider-util/strip-metabase-prefix)+ "unknown")]+ (map (fn [part]+ (if (= (:type part) :usage)+ (assoc part+ :model model+ :usage (-> (swap! usage-atom update model (partial merge-with +) (:usage part))+ (get model)))+ part)))))(defn- loop-step"Execute one iteration of the agent loop. Returns next loop state.@@ -470,7 +480,7 @@parts-atom (atom [])link-registry-atom (atom (get-in memory [:state :link-registry] {}))llm-call (call-llm memory context profile tools iteration tracking-opts link-registry-atom)- xf (comp (accumulate-usage-xf usage-atom)+ xf (comp (accumulate-usage-xf usage-atom (:model profile))(u/tee-xf parts-atom));; We use `reduce` instead of `transduce` because rf is the outer reducing;; function (e.g. aisdk-line-xf wrapping streaming-writer-rf) whose completiondiff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex 8359b0bbfdc..0728cb11462 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -20,6 +20,7 @@[metabase.metabot.context :as metabot.context][metabase.metabot.envelope :as metabot.envelope][metabase.metabot.feedback :as metabot.feedback]+ [metabase.metabot.provider-util :as provider-util][metabase.metabot.schema :as metabot.schema][metabase.metabot.self :as metabot.self][metabase.metabot.self.core :as self.core]@@ -48,7 +49,8 @@(= (:type %) "state"))messages)messages (-> (remove #(or (= % state) (= % finish)) messages)- vec)]+ vec)+ ai-proxy? (provider-util/metabase-provider? (metabot.settings/llm-metabot-provider))](app-db/update-or-insert! :model/MetabotConversation {:id conversation-id}(constantly (cond-> {:user_id api/*current-user-id*}state (assoc :state state))))@@ -64,7 +66,8 @@;; removed when ai-service does not give us `completionTokens` in `usage`(filter map?)(map #(+ (:prompt %) (:completion %)))- (apply +))})))+ (apply +))+ :ai_proxied (boolean ai-proxy?)})))(defn- extract-usage"Extract usage from parts, taking the last `:usage` per model.@@ -96,6 +99,7 @@(= "state" (:data-type %)))parts)usage (extract-usage parts)+ ai-proxy? (provider-util/metabase-provider? (metabot.settings/llm-metabot-provider));; Filter out :start, :usage, :finish, :data - these are metadata, not message content;; :data is like `:navigate_to`content (->> parts@@ -114,7 +118,8 @@:profile_id profile-id:total_tokens (->> (vals usage)(map #(+ (:prompt %) (:completion %)))- (reduce + 0))}))))+ (reduce + 0))+ :ai_proxied (boolean ai-proxy?)}))))(defn- streaming-writer-rf"Creates a reducing function that writes AI SDK lines to an OutputStream.@@ -278,7 +283,7 @@(throw e))))(def ^:private metabot-provider-schema- [:enum "anthropic" "openai" "openrouter"])+ (into [:enum] metabot.settings/supported-metabot-providers))(def ^:private llm-model-response-schema[:map@@ -400,9 +405,7 @@(defn- current-provider[]- (some-> (metabot.settings/llm-metabot-provider)- (str/split #"/" 2)- first))+ (provider-util/provider-and-model->provider (metabot.settings/llm-metabot-provider)))(defn- api-error->status-code[error]diff --git a/src/metabase/metabot/core.clj b/src/metabase/metabot/core.cljindex 8f0abfb7173..28fead05a7e 100644--- a/src/metabase/metabot/core.clj+++ b/src/metabase/metabot/core.clj@@ -1,13 +1,14 @@(ns metabase.metabot.core"API namespace for the `metabase.metabot` module."(:require- [metabase.metabot.api]+ [metabase.metabot.provider-util][metabase.metabot.scope][potemkin :as p]))(p/import-vars- [metabase.metabot.api- routes]+ [metabase.metabot.provider-util+ metabase-provider?+ provider-and-model->provider][metabase.metabot.scopeagent-metric-readagent-query-constructdiff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex dfe5a211643..f2c9e713c8f 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -8,7 +8,7 @@(defn store-message!"Persist messages to MetabotConversation and MetabotMessage tables."- [conversation-id profile-id messages & {:keys [slack-msg-id channel-id user-id]}]+ [conversation-id profile-id messages & {:keys [slack-msg-id channel-id user-id ai-proxy?]}](let [finish (let [m (u/last messages)](when (= (:_type m) :FINISH_MESSAGE)m))@@ -32,7 +32,8 @@;; removed when ai-service does not give us `completionTokens` in `usage`(filter map?)(map #(+ (:prompt %) (:completion %)))- (apply +))}+ (apply +))+ :ai_proxied (boolean ai-proxy?)}channel-id (assoc :channel_id channel-id)slack-msg-id (assoc :slack_msg_id slack-msg-id)user-id (assoc :user_id user-id)))))diff --git a/src/metabase/metabot/provider_util.clj b/src/metabase/metabot/provider_util.cljnew file mode 100644index 00000000000..6825eb314ca--- /dev/null+++ b/src/metabase/metabot/provider_util.clj@@ -0,0 +1,60 @@+(ns metabase.metabot.provider-util+ "Utility functions for parsing `provider-and-model` strings.++ These strings have the format `provider/model` (e.g. `anthropic/claude-haiku-4-5`,+ `openrouter/anthropic/claude-haiku-4-5`) or `metabase/provider/model` when routed+ through the Metabase Cloud AI proxy."+ (:require+ [clojure.string :as str]))++(def metabase-provider-prefix+ "The provider prefix that signals requests should be routed through the+ Metabase Cloud AI proxy."+ "metabase")++(defn metabase-provider?+ "Returns true when the provider-and-model string uses the Metabase AI proxy+ (i.e. starts with `metabase/`)."+ [provider-and-model]+ (boolean (some-> provider-and-model (str/starts-with? (str metabase-provider-prefix "/")))))++(defn provider-and-model->outer-provider+ "Extract the top-level provider prefix from a provider-and-model string.+ For `metabase/anthropic/model` returns `\"metabase\"`.+ For `anthropic/model` returns `\"anthropic\"`."+ [provider-and-model]+ (when provider-and-model+ (first (str/split provider-and-model #"/" 2))))++(defn provider-and-model->provider+ "Extract the direct provider from a provider-and-model string.+ For `metabase/anthropic/model` returns `\"anthropic\"`.+ For `anthropic/model` returns `\"anthropic\"`."+ [provider-and-model]+ (when provider-and-model+ (let [[first-seg rest-seg] (str/split provider-and-model #"/" 2)]+ (if (= first-seg metabase-provider-prefix)+ (first (str/split rest-seg #"/" 2))+ first-seg))))++(defn provider-and-model->model+ "Extract the model name from a provider-and-model string.+ For `metabase/anthropic/model` returns `\"model\"`.+ For `anthropic/model` returns `\"model\"`.+ For `openrouter/anthropic/claude-haiku-4-5` returns `\"anthropic/claude-haiku-4-5\"`."+ [provider-and-model]+ (when provider-and-model+ (let [[first-seg rest-seg] (str/split provider-and-model #"/" 2)]+ (if (= first-seg metabase-provider-prefix)+ (second (str/split rest-seg #"/" 2))+ rest-seg))))++(defn strip-metabase-prefix+ "Strip the `metabase/` routing prefix from a provider-and-model string.+ For `metabase/openrouter/anthropic/claude-haiku-4-5` returns+ `\"openrouter/anthropic/claude-haiku-4-5\"`.+ Returns the input unchanged if no prefix is present."+ [provider-and-model]+ (if (metabase-provider? provider-and-model)+ (str/replace-first provider-and-model (str metabase-provider-prefix "/") "")+ provider-and-model))diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 61c2e969ff7..8cf7b64c551 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -10,10 +10,10 @@TODO:- figure out what's lacking compared to ai-service"(:require- [clojure.string :as str][metabase.analytics.core :as analytics][metabase.analytics.prometheus :as prometheus][metabase.api.common :as api]+ [metabase.metabot.provider-util :as provider-util][metabase.metabot.self.claude :as claude][metabase.metabot.self.core :as core][metabase.metabot.self.openai :as openai]@@ -43,18 +43,19 @@{:provider provider}))))(defn- parse-provider-model [s]- (let [[prov model] (str/split s #"/" 2)]- {:provider prov- :stream-fn (resolve-adapter prov)- :model model}))+ (let [provider (provider-util/provider-and-model->provider s)]+ {:provider provider+ :stream-fn (resolve-adapter provider)+ :model (provider-util/provider-and-model->model s)+ :ai-proxy? (provider-util/metabase-provider? s)}))(defn list-models"List available models for a provider using its configured API key,or an override API key when provided."([provider]((resolve-model-lister provider)))- ([provider {:keys [api-key]}]- ((resolve-model-lister provider) api-key)))+ ([provider opts]+ ((resolve-model-lister provider) opts)));;; General LLM calling;; Matches the Python ai-service retry behavior:@@ -259,11 +260,11 @@([provider-and-model system-msg parts tools tracking-opts](call-llm provider-and-model system-msg parts tools tracking-opts nil))([provider-and-model system-msg parts tools tracking-opts {:keys [tool-choice]}]- (let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)]+ (let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)](log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools)- :tool-choice tool-choice})+ :tool-choice tool-choice :ai-proxy? ai-proxy?})(let [tracking-opts (assoc tracking-opts :model provider-and-model)- streaming-opts (cond-> {:model model :input parts :tools (vals tools)}+ streaming-opts (cond-> {:model model :input parts :tools (vals tools) :ai-proxy? ai-proxy?}system-msg (assoc :system system-msg)(and (seq tools)tool-choice) (assoc :tool_choice tool-choice))@@ -303,14 +304,15 @@Returns the parsed JSON map from the forced tool call."[provider-and-model messages json-schema temperature max-tokens tracking-opts]- (let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)+ (let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)_ (log/info "Calling LLM (structured)" {:provider provider :model model :msg-count (count messages)})tracking-opts (assoc tracking-opts :model provider-and-model)streaming-opts {:model model:input messages:schema json-schema:temperature temperature- :max-tokens max-tokens}]+ :max-tokens max-tokens+ :ai-proxy? ai-proxy?}](with-span :info {:name :metabot.agent/call-llm-structured:model model:msg-count (count messages)}diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.cljindex ef3284b5b58..f20f4a48289 100644--- a/src/metabase/metabot/self/claude.clj+++ b/src/metabase/metabot/self/claude.clj@@ -1,13 +1,12 @@(ns metabase.metabot.self.claude(:require- [clj-http.client :as http][clojure.string :as str][malli.json-schema :as mjs]- [metabase.llm.anthropic :as anthropic][metabase.llm.settings :as llm][metabase.metabot.self.core :as core][metabase.metabot.self.schema :as schema][metabase.util :as u]+ [metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.malli :as mu][metabase.util.o11y :refer [with-span]]))@@ -168,9 +167,7 @@merge-consecutivevec))-;;; ──────────────────────────────────────────────────────────────────;;; Tool definition format-;;; ──────────────────────────────────────────────────────────────────(defn- tool->claude"Convert a tool definition map to Claude API format.@@ -186,36 +183,43 @@:description doc:input_schema (mjs/transform params {:additionalProperties false})}))+(defn- anthropic-errors [res]+ (case (long (:status res 0))+ 401 (tru "Anthropic API key expired or invalid")+ 403 (tru "Anthropic API key has insufficient permissions")+ 404 (tru "Anthropic API endpoint is unavailable or the model was not found")+ 413 (tru "Anthropic API rejected our request because it was too large")+ 429 (tru "Anthropic API has rate limited us")+ 500 (tru "Anthropic API is not working but not saying why")+ 529 (tru "Anthropic API is overloaded and is asking us to wait")+ (tru "Unhandled error accessing Anthropic API")))+(defn list-models- "List available Anthropic models using the configured API key."- ([]- (list-models (llm/llm-anthropic-api-key)))- ([api-key]- (when (str/blank? api-key)- (throw (ex-info "No Anthropic API key is set" {:api-error true})))+ "List available Anthropic models.+ No-arg uses the configured API key. Opts map supports `:api-key` and `:ai-proxy?`."+ ([] (list-models {}))+ ([{:keys [api-key ai-proxy?]}]+ (when (and api-key (str/blank? api-key))+ (throw (core/missing-api-key-ex "Anthropic")))(try- (anthropic/list-models {:api-key api-key- :api-url (llm/llm-anthropic-api-base-url)- :api-version "2023-06-01"})+ (let [auth (core/resolve-auth "anthropic" "Anthropic"+ (when-let [k (or (not-empty api-key) (not-empty (llm/llm-anthropic-api-key)))]+ {:url (llm/llm-anthropic-api-base-url)+ :headers {"x-api-key" k}})+ ai-proxy?)+ res (core/request auth {:method :get+ :url "/v1/models"+ :headers {"anthropic-version" "2023-06-01"}})+ body (json/decode+kw (:body res))+ models (reverse (sort-by :created_at (:data body)))]+ {:models (map #(select-keys % [:id :display_name]) models)})(catch Exception e- (if-let [status (some-> e ex-data :status)]- (let [msg (case (int status)- 401 "Anthropic API key expired or invalid"- 403 "Anthropic API key has not enough permissions"- 404 "Anthropic API is telling us we cannot access this URL anymore?"- 429 "Anthropic API has rate limited us"- 500 "Anthropic API is not working but not saying why"- 529 "Anthropic API is overloaded and is asking us to wait"- "Unhandled error accessing Anthropic API")]- (throw (ex-info msg (assoc (ex-data e) :api-error true) e)))- (throw e))))))+ (core/rethrow-api-error! "anthropic" anthropic-errors e)))))(mu/defn claude-raw"Perform a streaming request to Claude API."- [{:keys [model system input tools schema tool_choice temperature max-tokens]+ [{:keys [model system input tools schema tool_choice temperature max-tokens ai-proxy?]:or {model "claude-haiku-4-5"}} :- core/LLMRequestOpts]- (when-not (llm/llm-anthropic-api-key)- (throw (ex-info "No Anthropic API key is set" {:api-error true})))(let [messages (parts->claude-messages input)all-tools (when (seq tools) (mapv tool->claude tools))req (cond-> {:model model@@ -239,33 +243,28 @@:msg-count (count input):tool-count (count tools)}(try- (let [res (http/post (str (llm/llm-anthropic-api-base-url) "/v1/messages")- {:as :stream- :headers {"x-api-key" (llm/llm-anthropic-api-key)- "anthropic-version" "2023-06-01"- "content-type" "application/json"}- :body (json/encode req)})]- (core/sse-reducible (:body res)))+ (let [api-key (not-empty (llm/llm-anthropic-api-key))+ auth (core/resolve-auth "anthropic" "Anthropic"+ (when api-key+ {:url (llm/llm-anthropic-api-base-url)+ :headers {"x-api-key" api-key}})+ ai-proxy?)+ response (core/request auth+ {:method :post+ :url "/v1/messages"+ :as :stream+ :headers {"anthropic-version" "2023-06-01"+ "content-type" "application/json"}+ :body (json/encode req)})]+ (core/sse-reducible (:body response)))(catch Exception e- (if-let [res (some-> (ex-data e)- json/decode-body)]- (let [status (:status res)- msg (case (int status)- 401 "Anthropic API key expired or invalid"- 403 "Anthropic API key has not enough permissions"- 404 "Anthropic API is telling us we cannot access this URL anymore?"- 413 "Anthropic API is not happy with our request being too big"- 429 "Anthropic API has rate limited us"- 500 "Anthropic API is not working but not saying why"- 529 "Anthropic API is overloaded and is asking us to wait"- "Unhandled error accessing Anthropic API")]- (throw (ex-info msg (assoc res :api-error true) e)))- (throw e)))))))+ (core/rethrow-api-error! "anthropic" anthropic-errors e))))))(defn claude"Call Claude API, return AISDK stream"[& args]- (eduction (claude->aisdk-chunks-xf) (apply claude-raw args)))+ (let [raw (apply claude-raw args)]+ (eduction (claude->aisdk-chunks-xf) raw)))(comment;; Now just use standard `into` - no core.async needed!diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex bc3e4b254a3..f42f3e83319 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -1,8 +1,12 @@(ns metabase.metabot.self.core(:require+ [clj-http.client :as http][clojure.java.io :as io][clojure.string :as str]+ [metabase.llm.settings :as llm]+ [metabase.premium-features.core :as premium-features][metabase.util :as u]+ [metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log][metabase.util.o11y :refer [with-span]])@@ -41,7 +45,8 @@:temperature - Sampling temperature:max-tokens - Maximum tokens in the response:schema - JSON Schema map for structured output; each provider forces a- tool call (Claude, OpenRouter) or uses json_schema mode (OpenAI)"+ tool call (Claude, OpenRouter) or uses json_schema mode (OpenAI)+ :ai-proxy? - When true, skip provider auth and use the Metabase AI proxy"[:map[:model {:optional true} :string][:system {:optional true} [:maybe :string]]@@ -50,7 +55,8 @@[:tool_choice {:optional true} [:maybe [:enum "auto" "required"]]][:temperature {:optional true} [:maybe number?]][:max-tokens {:optional true} [:maybe :int]]- [:schema {:optional true} :any]])+ [:schema {:optional true} :any]+ [:ai-proxy? {:optional true} [:maybe :boolean]]])(defn mkid"Generate a random id"@@ -480,3 +486,54 @@nil)(rf result chunk))))))++(defn rethrow-api-error!+ "Rethrow a provider HTTP exception with a translated, user-facing message.++ `res->message` receives the decoded response map and must return the message+ to surface to the client. If the exception already carries `:api-error true`+ in its ex-data (e.g. a missing-API-key error from [[resolve-auth]]) it is+ rethrown as-is so the original message is preserved."+ [provider res->message e]+ (let [data (ex-data e)]+ (cond+ (:api-error data) (throw e)+ (:body data) (let [res (json/decode-body data)]+ (throw (ex-info (res->message res)+ (assoc res+ :api-error true+ :provider provider+ :error-code :provider-api-error)+ e)))+ :else (throw e))))++(defn missing-api-key-ex+ "Create a standardized missing-API-key exception for provider adapters."+ [llm-type]+ (ex-info (tru "No {0} API key is set" llm-type)+ {:api-error true+ :error-code :api-key-missing}))++(defn resolve-auth+ "Pick the right auth map for an LLM request.++ - When `ai-proxy?` is true, uses the Metabase Cloud proxy (errors if unconfigured).+ - Otherwise uses the provider's BYOK `auth`."+ [provider-slug llm-type auth ai-proxy?]+ (let [proxy-auth (when-let [base (llm/llm-proxy-base-url)]+ {:url (str (str/replace base #"/+$" "") "/" provider-slug)+ :headers {"x-metabase-instance-token" (premium-features/premium-embedding-token)}})]+ (if ai-proxy?+ (or proxy-auth+ (throw (ex-info (tru "AI proxy is not configured")+ {:api-error true+ :error-code :proxy-not-configured})))+ (or auth+ (throw (missing-api-key-ex llm-type))))))++(defn request+ "Perform an LLM HTTP request with the given auth (a map of `:url` and `:headers`)."+ [{:keys [url headers]} req]+ (http/request (-> req+ (update :url #(str url %))+ (update :headers merge headers))))diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.cljindex ab962b6d0a8..5d1fb62faae 100644--- a/src/metabase/metabot/self/openai.clj+++ b/src/metabase/metabot/self/openai.clj@@ -1,12 +1,12 @@(ns metabase.metabot.self.openai(:require- [clj-http.client :as http][clojure.string :as str][malli.json-schema :as mjs][metabase.llm.settings :as llm][metabase.metabot.self.core :as core][metabase.metabot.self.schema :as schema][metabase.util :as u]+ [metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.malli :as mu]))@@ -103,7 +103,9 @@:inputTextDelta delta}))(= (:type chunk)"response.completed") (rf {:type :usage- :usage (:usage response)+ :usage (let [u (:usage response)]+ {:promptTokens (:input_tokens u 0)+ :completionTokens (:output_tokens u 0)});; non-standard extension, not in AISDK5:id (:id response):model @model-name})@@ -157,41 +159,43 @@:description doc:parameters (mjs/transform params {:additionalProperties false})}))+(defn- openai-errors [res]+ (case (long (:status res 0))+ 401 (tru "OpenAI API key expired or invalid")+ 403 (tru "OpenAI API key has insufficient permissions")+ 404 (tru "OpenAI API endpoint or model listing is unavailable")+ 429 (tru "OpenAI API has rate limited us")+ 500 (tru "OpenAI API is not working but not saying why")+ (tru "Unhandled error accessing OpenAI API")))+(defn list-models- "List available OpenAI models using the configured API key."- ([]- (list-models (llm/llm-openai-api-key)))- ([api-key]- (when (str/blank? api-key)- (throw (ex-info "No OpenAI API key is set" {:api-error true})))+ "List available OpenAI models.+ No-arg uses the configured API key. Opts map supports `:api-key` and `:ai-proxy?`."+ ([] (list-models {}))+ ([{:keys [api-key ai-proxy?]}]+ (when (and api-key (str/blank? api-key))+ (throw (core/missing-api-key-ex "OpenAI")))(try- (let [res (http/get (str (llm/llm-openai-api-base-url) "/v1/models")- {:as :json- :headers {"Authorization" (str "Bearer " api-key)- "Content-Type" "application/json"}})]+ (let [auth (core/resolve-auth "openai" "OpenAI"+ (when-let [k (or (not-empty api-key) (not-empty (llm/llm-openai-api-key)))]+ {:url (llm/llm-openai-api-base-url)+ :headers {"Authorization" (str "Bearer " k)}})+ ai-proxy?)+ res (core/request auth {:method :get+ :url "/v1/models"+ :as :json+ :headers {"Content-Type" "application/json"}})]{:models (mapv (fn [model]{:id (:id model):display_name (:id model)})(reverse (sort-by :created (get-in res [:body :data]))))})(catch Exception e- (if-let [res (some-> (ex-data e) json/decode-body)]- (let [status (:status res)- msg (case (int status)- 401 "OpenAI API key expired or invalid"- 403 "OpenAI API key has insufficient permissions"- 404 "OpenAI API endpoint or model listing is unavailable"- 429 "OpenAI API has rate limited us"- 500 "OpenAI API is not working but not saying why"- "Unhandled error accessing OpenAI API")]- (throw (ex-info msg (assoc res :api-error true) e)))- (throw e))))))+ (core/rethrow-api-error! "openai" openai-errors e)))))(mu/defn openai-raw"Perform a streaming request to OpenAI Responses API."- [{:keys [model system input tools schema tool_choice temperature max-tokens]+ [{:keys [model system input tools schema tool_choice temperature max-tokens ai-proxy?]:or {model "gpt-4.1-mini"}} :- core/LLMRequestOpts]- (when-not (llm/llm-openai-api-key)- (throw (ex-info "No OpenAI API key is set" {:api-error true})))(let [all-tools (or (when schema;; Structured output: force a tool call with the given JSON schema[{:type "function"@@ -212,18 +216,24 @@temperature (assoc :temperature temperature)max-tokens (assoc :max_tokens max-tokens))](try- (let [res (http/post (str (llm/llm-openai-api-base-url) "/v1/responses")- {:as :stream- :headers {"Authorization" (str "Bearer " (llm/llm-openai-api-key))- "Content-Type" "application/json"}- :body (json/encode req)})]- (core/sse-reducible (:body res)))+ (let [api-key (not-empty (llm/llm-openai-api-key))+ auth (core/resolve-auth "openai" "OpenAI"+ (when api-key+ {:url (llm/llm-openai-api-base-url)+ :headers {"Authorization" (str "Bearer " api-key)}})+ ai-proxy?)+ response (core/request auth+ {:method :post+ :url "/v1/responses"+ :as :stream+ :headers {"Content-Type" "application/json"}+ :body (json/encode req)})]+ (core/sse-reducible (:body response)))(catch Exception e- (if-let [res (ex-data e)]- (throw (ex-info (.getMessage e) (json/decode-body res)))- (throw e))))))+ (core/rethrow-api-error! "openai" openai-errors e)))))(defn openai"Call OpenAI API, return AISDK stream."[& args]- (eduction (openai->aisdk-chunks-xf) (apply openai-raw args)))+ (let [raw (apply openai-raw args)]+ (eduction (openai->aisdk-chunks-xf) raw)))diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.cljindex e248ff17a76..e2165929691 100644--- a/src/metabase/metabot/self/openrouter.clj+++ b/src/metabase/metabot/self/openrouter.clj@@ -8,13 +8,13 @@The agent loop produces AISDK parts as its canonical message format. Thisadapter converts those directly to Chat Completions messages."(:require- [clj-http.client :as http][clojure.string :as str][malli.json-schema :as mjs][metabase.llm.settings :as llm][metabase.metabot.self.core :as core][metabase.metabot.self.schema :as schema][metabase.util :as u]+ [metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]@@ -91,39 +91,44 @@:description doc:parameters (mjs/transform params {:additionalProperties false})}}))+(defn- openrouter-errors [res]+ (case (long (:status res 0))+ 401 (tru "OpenRouter API key expired or invalid")+ 402 (tru "OpenRouter has insufficient credits")+ 403 (tru "OpenRouter API key has insufficient permissions")+ 404 (tru "OpenRouter model listing endpoint is unavailable")+ 429 (tru "OpenRouter has rate limited us")+ 500 (tru "OpenRouter returned an internal server error")+ 502 (tru "OpenRouter upstream provider returned an error")+ 503 (tru "OpenRouter service is unavailable")+ (or (-> res :body :error :message)+ (tru "Unhandled error accessing OpenRouter API"))))+(defn list-models- "List available OpenRouter models using the configured API key."- ([]- (list-models (llm/llm-openrouter-api-key)))- ([api-key]- (when (str/blank? api-key)- (throw (ex-info "No OpenRouter API key is set" {:api-error true})))+ "List available OpenRouter models.+ No-arg uses the configured API key. Opts map supports `:api-key` and `:ai-proxy?`."+ ([] (list-models {}))+ ([{:keys [api-key ai-proxy?]}]+ (when (and api-key (str/blank? api-key))+ (throw (core/missing-api-key-ex "OpenRouter")))(try- (let [res (http/get (str (llm/llm-openrouter-api-base-url) "/v1/models")- {:as :json- :headers {"Authorization" (str "Bearer " api-key)- "Content-Type" "application/json"- "HTTP-Referer" "https://metabase.com"- "X-Title" "Metabase"}})]+ (let [auth (core/resolve-auth "openrouter" "OpenRouter"+ (when-let [k (or (not-empty api-key) (not-empty (llm/llm-openrouter-api-key)))]+ {:url (llm/llm-openrouter-api-base-url)+ :headers {"Authorization" (str "Bearer " k)}})+ ai-proxy?)+ res (core/request auth {:method :get+ :url "/v1/models"+ :as :json+ :headers {"Content-Type" "application/json"+ "HTTP-Referer" "https://metabase.com"+ "X-Title" "Metabase"}})]{:models (mapv (fn [model]{:id (:id model):display_name (or (:name model) (:id model))})(reverse (sort-by :created (get-in res [:body :data]))))})(catch Exception e- (if-let [res (some-> (ex-data e) json/decode-body)]- (let [status (:status res)- msg (case (int status)- 401 "OpenRouter API key expired or invalid"- 402 "OpenRouter: insufficient credits"- 403 "OpenRouter API key has insufficient permissions"- 404 "OpenRouter: model listing endpoint unavailable"- 429 "OpenRouter: rate limited"- 500 "OpenRouter: internal server error"- 502 "OpenRouter: upstream provider error"- 503 "OpenRouter: service unavailable"- "Unhandled error accessing OpenRouter API")]- (throw (ex-info msg (assoc res :api-error true) e)))- (throw e))))))+ (core/rethrow-api-error! "openrouter" openrouter-errors e)))));;; Streaming response → AISDK v5 chunks@@ -248,62 +253,54 @@Works with OpenRouter, or any OpenAI-compatible endpoint that supports`/v1/chat/completions` (e.g. vLLM, Ollama, Together, etc.)."- [{:keys [model system input tools temperature max-tokens tool_choice schema]+ [{:keys [model system input tools temperature max-tokens tool_choice schema ai-proxy?]:or {model "anthropic/claude-haiku-4-5"}} :- core/LLMRequestOpts]- (when-not (llm/llm-openrouter-api-key)- (throw (ex-info "No OpenRouter API key is set" {:api-error true})))- (let [messages (cond-> (parts->cc-messages input)- system (as-> msgs (into [{:role "system" :content system}] msgs)))- all-tools (or (when schema- ;; Structured output: force a tool call with the given JSON schema- [{:type "function"- :function {:name "structured_output"- :description "Output structured data"- :parameters schema}}])- (seq (mapv tool->openai-chat tools)))- req (cond-> {:model model- :stream true- :stream_options {:include_usage true}- :messages messages}- all-tools (assoc :tools (vec all-tools)- :tool_choice (cond- schema "required"- tool_choice tool_choice- :else "auto"))- temperature (assoc :temperature temperature)- max-tokens (assoc :max_tokens max-tokens))]+ (let [messages (cond-> (parts->cc-messages input)+ system (as-> msgs (into [{:role "system" :content system}] msgs)))+ all-tools (or (when schema+ ;; Structured output: force a tool call with the given JSON schema+ [{:type "function"+ :function {:name "structured_output"+ :description "Output structured data"+ :parameters schema}}])+ (seq (mapv tool->openai-chat tools)))+ req (cond-> {:model model+ :stream true+ :stream_options {:include_usage true}+ :messages messages}+ all-tools (assoc :tools (vec all-tools)+ :tool_choice (cond+ schema "required"+ tool_choice tool_choice+ :else "auto"))+ temperature (assoc :temperature temperature)+ max-tokens (assoc :max_tokens max-tokens))](log/debug "OpenRouter request" {:model model :msg-count (count messages) :tools (count (or tools []))})(with-span :info {:name :metabot.openrouter/request:model model:msg-count (count messages):tool-count (count (or tools []))}(try- (let [res (http/post (str (llm/llm-openrouter-api-base-url) "/v1/chat/completions")- {:as :stream- :headers {"Authorization" (str "Bearer " (llm/llm-openrouter-api-key))- "Content-Type" "application/json"- "HTTP-Referer" "https://metabase.com"- "X-Title" "Metabase"}- :body (json/encode req)})]- (core/sse-reducible (:body res)))+ (let [api-key (not-empty (llm/llm-openrouter-api-key))+ auth (core/resolve-auth "openrouter" "OpenRouter"+ (when api-key+ {:url (llm/llm-openrouter-api-base-url)+ :headers {"Authorization" (str "Bearer " api-key)}})+ ai-proxy?)+ response (core/request auth+ {:method :post+ :url "/v1/chat/completions"+ :as :stream+ :headers {"Content-Type" "application/json"+ "HTTP-Referer" "https://metabase.com"+ "X-Title" "Metabase"}+ :body (json/encode req)})]+ (core/sse-reducible (:body response)))(catch Exception e- (if-let [res (some-> (ex-data e) json/decode-body)]- (let [status (:status res)- msg (case (int status)- 401 "OpenRouter API key expired or invalid"- 402 "OpenRouter: insufficient credits"- 403 "OpenRouter API key has insufficient permissions"- 404 "OpenRouter: model not found or endpoint unavailable"- 429 "OpenRouter: rate limited"- 500 "OpenRouter: internal server error"- 502 "OpenRouter: upstream provider error"- 503 "OpenRouter: service unavailable"- (or (-> res :body :error :message)- "Unhandled error accessing OpenRouter API"))]- (throw (ex-info msg (assoc res :api-error true) e)))- (throw e)))))))+ (core/rethrow-api-error! "openrouter" openrouter-errors e))))))(defn openrouter"Call OpenRouter Chat Completions API, return AISDK stream."[& args]- (eduction (openrouter->aisdk-chunks-xf) (apply openrouter-raw args)))+ (let [raw (apply openrouter-raw args)]+ (eduction (openrouter->aisdk-chunks-xf) raw)))diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex 31f7e23444f..5c05d49d720 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][metabase.llm.settings :as llm.settings]+ [metabase.metabot.provider-util :as provider-util][metabase.settings.core :as setting :refer [defsetting]][metabase.util.i18n :refer [deferred-tru tru]]))@@ -91,28 +92,47 @@;;; ------------------------------------------------- LLM Provider -------------------------------------------------(def ^:private supported-metabot-providers+(def supported-metabot-providers"Set of supported LLM provider prefixes for the `llm-metabot-provider` setting."+ #{"anthropic" "openai" "openrouter" provider-util/metabase-provider-prefix})++(def ^:private direct-providers+ "Providers that can be used directly (not via the metabase/ proxy prefix)."#{"anthropic" "openai" "openrouter"})(defn- validate-metabot-provider!"Validate that `value` has the format `provider/model` with a supported provider prefix.+ For `metabase/` prefix, validates the inner provider too (e.g. `metabase/anthropic/model`).Throws an exception with `:status-code 400` on invalid input."[value](when-not (string? value)(throw (ex-info (tru "Metabot provider must be a string, got: {0}" (pr-str value)){:status-code 400})))- (let [[provider model] (str/split value #"/" 2)]- (when-not (contains? supported-metabot-providers provider)+ (let [outer-provider (provider-util/provider-and-model->outer-provider value)]+ (when-not (contains? supported-metabot-providers outer-provider)(throw (ex-info (tru "Unknown provider {0}. Supported providers: {1}"- (pr-str provider) (str/join ", " (sort supported-metabot-providers)))+ (pr-str outer-provider) (str/join ", " (sort supported-metabot-providers))){:status-code 400- :provider provider+ :provider outer-provider:supported supported-metabot-providers})))- (when (str/blank? model)+ (when (str/blank? (provider-util/provider-and-model->model value))(throw (ex-info (tru "Model name is required. Expected format: provider/model, e.g. \"anthropic/claude-haiku-4-5\""){:status-code 400- :value value})))))+ :value value})))+ ;; For metabase/ prefix, validate the inner provider+ (when (= outer-provider provider-util/metabase-provider-prefix)+ (let [inner-provider (provider-util/provider-and-model->provider value)+ inner-model (provider-util/provider-and-model->model value)]+ (when-not (contains? direct-providers inner-provider)+ (throw (ex-info (tru "Unknown inner provider {0} in metabase/ prefix. Supported: {1}"+ (pr-str inner-provider) (str/join ", " (sort direct-providers)))+ {:status-code 400+ :provider inner-provider+ :supported direct-providers})))+ (when (str/blank? inner-model)+ (throw (ex-info (tru "Model name is required. Expected format: metabase/provider/model, e.g. \"metabase/anthropic/claude-haiku-4-5\"")+ {:status-code 400+ :value value})))))))(defsetting llm-metabot-provider(deferred-tru "The AI provider and model for Metabot. Format: provider/model-name, e.g. `anthropic/claude-haiku-4-5`, `openai/gpt-4.1-mini`, `openrouter/anthropic/claude-haiku-4-5`.")@@ -156,16 +176,18 @@"openrouter" (llm.settings/llm-openrouter-api-key)nil))-(defn- provider-prefix [provider-and-model]- (some-> provider-and-model- (str/split #"/" 2)- first))--(defn- llm-provider-configured? [provider-and-model]- (boolean (some-> provider-and-model- provider-prefix- configured-provider-api-key- token-configured?)))+(defn- llm-provider-configured?+ "Check if a provider-and-model string has the necessary configuration.+ For `metabase/*` providers, checks that the proxy URL is set.+ For direct providers, checks that a BYOK API key is set."+ [provider-and-model]+ (boolean+ (if (provider-util/metabase-provider? provider-and-model)+ (some? (llm.settings/llm-proxy-base-url))+ (some-> provider-and-model+ provider-util/provider-and-model->provider+ configured-provider-api-key+ token-configured?))))(defsetting llm-metabot-internal-tasks-enabled?(deferred-tru "Controls whether Metabot performs internal tasks that might require background tasks or additional LLM calls (e.g. user intent classification).")diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.cljindex 5ee7fe1ca20..3893678ac13 100644--- a/src/metabase/slackbot/streaming.clj+++ b/src/metabase/slackbot/streaming.clj@@ -8,9 +8,11 @@[metabase.channel.slack :as channel.slack][metabase.metabot.agent.core :as agent][metabase.metabot.context :as metabot.context]+ [metabase.metabot.core :as metabot][metabase.metabot.envelope :as metabot.envelope][metabase.metabot.persistence :as metabot.persistence][metabase.metabot.self.core :as self.core]+ [metabase.metabot.settings :as metabot.settings][metabase.metabot.util :as metabot.u][metabase.permissions.core :as perms][metabase.slackbot.channel :as slackbot.channel]@@ -181,7 +183,8 @@messages (conj (vec history) request-message)_ (metabot.persistence/store-message! conversation-id "slackbot" [message]:channel-id channel-id- :slack-msg-id req-slack-msg-id)+ :slack-msg-id req-slack-msg-id+ :ai-proxy? (metabot/metabase-provider? (metabot.settings/llm-metabot-provider)))parts-atom (atom [])dispatch-xf (comp(u/tee-xf parts-atom)@@ -219,13 +222,15 @@:state {}:profile-id :slackbot:context context}))- (let [lines (into [] (self.core/aisdk-line-xf) @parts-atom)- pk (metabot.persistence/store-message!- conversation-id "slackbot"- (metabot.u/aisdk->messages :assistant lines)- :channel-id channel-id- :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))- :user-id api/*current-user-id*)]+ (let [parts @parts-atom+ lines (into [] (self.core/aisdk-line-xf) parts)+ pk (metabot.persistence/store-message!+ conversation-id "slackbot"+ (metabot.u/aisdk->messages :assistant lines)+ :channel-id channel-id+ :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))+ :user-id api/*current-user-id*+ :ai-proxy? (metabot/metabase-provider? (metabot.settings/llm-metabot-provider)))](when stored-msg-id(reset! stored-msg-id pk)))))diff --git a/test/metabase/internal_stats/metabot_test.clj b/test/metabase/internal_stats/metabot_test.cljnew file mode 100644index 00000000000..c1905f78489--- /dev/null+++ b/test/metabase/internal_stats/metabot_test.clj@@ -0,0 +1,141 @@+(ns metabase.internal-stats.metabot-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [java-time.api :as t]+ [metabase.internal-stats.metabot :as sut]+ [metabase.metabot.self.openrouter :as openrouter]+ [metabase.metabot.settings :as metabot.settings]+ [metabase.metabot.test-util :as mut]+ [metabase.search.test-util :as search.tu]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++;; ---------------------------------------------------------------------------+;; Helpers+;; ---------------------------------------------------------------------------++(defn- send-message!+ "Send a message through the real streaming endpoint, returning the HTTP response body.+ The LLM is mocked to return a simple text response with the given model/usage."+ [conversation-id message model prompt-tokens completion-tokens]+ (with-redefs [openrouter/openrouter+ (fn [_]+ (mut/mock-llm-response+ [{:type :start :id "msg-1"}+ {:type :text :text "Hello!"}+ {:type :usage+ :model model+ :usage {:promptTokens prompt-tokens :completionTokens completion-tokens}+ :id "msg-1"}]))]+ (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ {:message message+ :context {}+ :conversation_id conversation-id+ :history []+ :state {}})))++(defn- backdate-messages!+ "Update created_at on all messages for a conversation to the given timestamp."+ [conversation-id created-at]+ (t2/update! :model/MetabotMessage {:conversation_id conversation-id}+ {:created_at created-at}))++(defn- cleanup! [& conv-ids]+ (doseq [cid conv-ids]+ (t2/delete! :model/MetabotMessage :conversation_id cid)+ (t2/delete! :model/MetabotConversation :id cid)))++;; ---------------------------------------------------------------------------+;; Tests+;; ---------------------------------------------------------------------------++(deftest metabot-stats-e2e-test+ (search.tu/with-index-disabled+ (let [clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")+ yesterday (t/offset-date-time 2026 3 31 10 30 0 0 (t/zone-offset "+00"))+ today (t/offset-date-time 2026 4 1 9 0 0 0 (t/zone-offset "+00"))+ two-days-ago (t/offset-date-time 2026 3 29 14 0 0 0 (t/zone-offset "+00"))+ conv-1 (str (random-uuid))+ conv-2 (str (random-uuid))+ conv-3 (str (random-uuid))+ conv-4 (str (random-uuid))+ conv-5 (str (random-uuid))+ all-convs [conv-1 conv-2 conv-3 conv-4 conv-5]+ ;; OpenRouter returns model names like "anthropic/claude-haiku-4-5" in its API.+ ;; This is the bare model name that flows from the SSE adapter → extract-usage → DB.+ model "anthropic/claude-haiku-4-5"]+ (t/with-clock clock+ (try+ ;; -- AI proxy conversations (metabase/ prefix) --+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider+ "metabase/openrouter/anthropic/claude-haiku-4-5"]+ ;; conv-1: yesterday, one model+ (send-message! conv-1 "What is 2+2?" model 100 50)+ (backdate-messages! conv-1 yesterday)++ ;; conv-2: yesterday, same model different usage+ (send-message! conv-2 "Tell me a joke" model 200 80)+ (backdate-messages! conv-2 yesterday)++ ;; conv-3: two days ago — out of window+ (send-message! conv-3 "Old question" model 999 999)+ (backdate-messages! conv-3 two-days-ago)++ ;; conv-4: today — out of window+ (send-message! conv-4 "Today's question" model 888 888)+ (backdate-messages! conv-4 today))++ ;; -- BYOK conversation (no metabase/ prefix) --+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider+ "openrouter/anthropic/claude-haiku-4-5"]+ (send-message! conv-5 "BYOK question" model 777 777)+ (backdate-messages! conv-5 yesterday))++ ;; -- Verify stored data --++ (testing "ai-proxy messages have ai_proxied = true on ALL rows (user + assistant)"+ (let [msgs (t2/select :model/MetabotMessage :conversation_id conv-1)]+ (is (= 2 (count msgs)) "should have user + assistant messages")+ (is (every? true? (map :ai_proxied msgs)))))++ (testing "BYOK messages have ai_proxied = false on ALL rows"+ (let [msgs (t2/select :model/MetabotMessage :conversation_id conv-5)]+ (is (= 2 (count msgs)))+ (is (every? false? (map :ai_proxied msgs)))))++ (testing "usage keys are provider/model (metabase/ prefix stripped)"+ ;; accumulate-usage-xf strips metabase/ prefix → "openrouter/anthropic/claude-haiku-4-5"+ ;; JSON roundtrip keywordizes → :openrouter/anthropic/claude-haiku-4-5+ (let [msg (t2/select-one :model/MetabotMessage :conversation_id conv-1 :role :assistant)]+ (is (contains? (:usage msg) (keyword "openrouter" "anthropic/claude-haiku-4-5"))+ "usage key should be provider/model without metabase/ prefix")))++ ;; -- Verify stats aggregation --++ (let [stats (sut/metabot-stats)]+ (testing ":metabot-tokens sums total_tokens for yesterday's ai-proxied messages only"+ ;; conv-1: 150, conv-2: 280 → total 430+ (is (= 430 (:metabot-tokens stats))))++ (testing ":metabot-usage aggregates by model:in/out"+ (is (= {"openrouter:anthropic/claude-haiku-4-5:in" 300+ "openrouter:anthropic/claude-haiku-4-5:out" 130}+ (:metabot-usage stats))))++ (testing ":metabot-queries counts ai-proxied user messages for yesterday"+ (is (= 2 (:metabot-queries stats))))++ (testing ":metabot-users counts distinct users for yesterday"+ (is (= 1 (:metabot-users stats))))++ (testing ":metabot-usage-date is yesterday's date"+ (is (= "2026-03-31" (:metabot-usage-date stats)))))++ ;; -- BYOK-only scenario --+ (cleanup! conv-1 conv-2 conv-3 conv-4)++ (testing "returns nil when only BYOK (non-proxied) messages exist yesterday"+ (is (nil? (sut/metabot-stats))))++ (finally+ (apply cleanup! all-convs)))))))diff --git a/test/metabase/llm/anthropic_test.clj b/test/metabase/llm/anthropic_test.cljindex b3f095851eb..00c9ba24faf 100644--- a/test/metabase/llm/anthropic_test.clj+++ b/test/metabase/llm/anthropic_test.clj@@ -3,8 +3,8 @@[clj-http.client :as http][clojure.test :refer :all][metabase.llm.anthropic :as anthropic]- [metabase.test :as mt]- [metabase.util.json :as json]))+ [metabase.llm.settings :as llm.settings]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -106,7 +106,7 @@(deftest chat-completion-not-configured-test(testing "throws when API key not configured"- (mt/with-temporary-setting-values [llm-anthropic-api-key nil]+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)](is (thrown-with-msg?clojure.lang.ExceptionInfo#"not configured"@@ -135,40 +135,3 @@:prompt 1500:completion 250}}result))))))))--;;; ------------------------------------------- list-models Tests ---------------------------------------------(deftest list-models-requires-api-key-test- (testing "list-models throws when API key is not configured"- (mt/with-temporary-setting-values [llm-anthropic-api-key nil]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"not configured"- (anthropic/list-models))))))--(deftest list-models-success-test- (testing "list-models returns sorted models with selected fields"- (let [mock-response-body {:data [{:id "claude-sonnet-4-20250514"- :display_name "Claude Sonnet 4"- :created_at "2025-05-14T00:00:00Z"- :type "model"}- {:id "claude-sonnet-4-5-20250929"- :display_name "Claude Sonnet 4.5"- :created_at "2025-09-29T00:00:00Z"- :type "model"}- {:id "claude-opus-4-5-20251101"- :display_name "Claude Opus 4.5"- :created_at "2025-11-01T00:00:00Z"- :type "model"}]- :first_id "claude-sonnet-4-20250514"- :last_id "claude-opus-4-5-20251101"- :has_more false}]- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"]- (with-redefs [http/get (constantly {:status 200- :body (json/encode mock-response-body)})]- (let [result (anthropic/list-models)]- ;; Should be sorted by created_at descending (newest first)- (is (= [{:id "claude-opus-4-5-20251101" :display_name "Claude Opus 4.5"}- {:id "claude-sonnet-4-5-20250929" :display_name "Claude Sonnet 4.5"}- {:id "claude-sonnet-4-20250514" :display_name "Claude Sonnet 4"}]- (:models result)))))))))diff --git a/test/metabase/llm/api_test.clj b/test/metabase/llm/api_test.cljindex 29415d0bca7..eb3f4142188 100644--- a/test/metabase/llm/api_test.clj+++ b/test/metabase/llm/api_test.clj@@ -7,6 +7,9 @@[metabase.llm.anthropic :as llm.anthropic][metabase.llm.api :as api][metabase.llm.context :as llm.context]+ [metabase.llm.settings :as llm.settings]+ [metabase.metabot.self :as metabot.self]+ [metabase.metabot.settings :as metabot.settings][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -125,14 +128,14 @@(deftest generate-sql-error-handling-test(mt/with-temp [:model/Database db {:engine :postgres}](testing "403 when LLM not configured"- (mt/with-temporary-setting-values [llm-anthropic-api-key nil]+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)](let [response (mt/user-http-request :rasta :post 403 "llm/generate-sql"{:prompt "test":database_id (:id db)})](is (str/includes? (str response) "not configured")))))(testing "400 when no tables found"- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"]+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly "sk-ant-test")](let [response (mt/user-http-request :rasta :post 400 "llm/generate-sql"{:prompt "no table mentions here":database_id (:id db)})]@@ -140,10 +143,42 @@(deftest list-models-unconfigured-test(testing "Returns 403 when LLM is not configured"- (mt/with-temporary-setting-values [llm-anthropic-api-key nil]+ (with-redefs [metabot.settings/llm-metabot-configured? (constantly false)](let [response (mt/user-http-request :rasta :get 403 "llm/list-models")](is (str/includes? (str response) "not configured"))))))+(deftest list-models-resolves-direct-provider-test+ (testing "resolves provider from provider-and-model and passes ai-proxy? = false"+ (let [captured (atom nil)]+ (with-redefs [metabot.settings/llm-metabot-configured? (constantly true)+ metabot.settings/llm-metabot-provider (constantly "anthropic/claude-sonnet-4")+ metabot.self/list-models (fn [provider opts]+ (reset! captured {:provider provider :opts opts})+ {:models [{:id "claude-sonnet-4" :display_name "Claude Sonnet 4"}]})]+ (let [response (mt/user-http-request :rasta :get 200 "llm/list-models")]+ (is (=? {:models [{:id "claude-sonnet-4"+ :display_name "Claude Sonnet 4"}]}+ response))+ (is (=? {:provider "anthropic"+ :opts {:ai-proxy? false}}+ @captured)))))))++(deftest list-models-resolves-metabase-prefixed-provider-test+ (testing "resolves inner provider from metabase/ prefix and passes ai-proxy? = true"+ (let [captured (atom nil)]+ (with-redefs [metabot.settings/llm-metabot-configured? (constantly true)+ metabot.settings/llm-metabot-provider (constantly "metabase/openrouter/anthropic/claude-haiku-4-5")+ metabot.self/list-models (fn [provider opts]+ (reset! captured {:provider provider :opts opts})+ {:models [{:id "anthropic/claude-haiku-4-5" :display_name "Claude Haiku 4.5"}]})]+ (let [response (mt/user-http-request :rasta :get 200 "llm/list-models")]+ (is (=? {:models [{:id "anthropic/claude-haiku-4-5"+ :display_name "Claude Haiku 4.5"}]}+ response))+ (is (=? {:provider "openrouter"+ :opts {:ai-proxy? true}}+ @captured)))))))+;;; ------------------------------------------- Snowplow Tests -------------------------------------------(defn- token-usage-event? [event]@@ -168,54 +203,54 @@:prompt 1000:completion 200}:duration-ms 500}]- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"]- (snowplow-test/with-fake-snowplow-collector- (with-redefs [llm.anthropic/chat-completion (constantly mock-chat-response)]- (let [response (mt/user-http-request :rasta :post 200 "llm/generate-sql"- {:prompt "get all users"- :database_id (:id db)- :referenced_entities [{:model "table" :id (:id table)}]})- events (snowplow-test/pop-event-data-and-user-id!)- token-events (filter token-usage-event? events)- simple-events (filter simple-event? events)]- (is (= "SELECT * FROM users" (:sql response)))- (testing "token_usage event"- (is (=? [{:data {"model_id" "claude-sonnet-4-5-20250929"- "prompt_tokens" 1000- "completion_tokens" 200- "total_tokens" 1200- "estimated_costs_usd" 0.0- "duration_ms" 500- "source" "oss_metabot"- "tag" "oss-sqlgen"}}]- token-events)))- (testing "simple_event"- (is (=? [{:data {"event" "metabot_oss_sqlgen_used"- "duration_ms" int?- "result" "success"- "event_detail" "postgres"}}]- simple-events)))))))))))--(deftest generate-sql-snowplow-failure-test- (testing "failed /generate-sql call tracks simple_event with failure result"- (mt/with-temp [:model/Database db {:engine :postgres}- :model/Table table {:db_id (:id db) :name "users" :schema "public"}- :model/Field _ {:table_id (:id table) :name "id" :base_type :type/Integer}]- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"](snowplow-test/with-fake-snowplow-collector- (with-redefs [llm.anthropic/chat-completion (fn [_] (throw (Exception. "API error")))]- (mt/user-http-request :rasta :post 500 "llm/generate-sql"- {:prompt "get all users"- :database_id (:id db)- :referenced_entities [{:model "table" :id (:id table)}]})- (let [events (snowplow-test/pop-event-data-and-user-id!)+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly "sk-ant-test")+ llm.anthropic/chat-completion (constantly mock-chat-response)]+ (let [response (mt/user-http-request :rasta :post 200 "llm/generate-sql"+ {:prompt "get all users"+ :database_id (:id db)+ :referenced_entities [{:model "table" :id (:id table)}]})+ events (snowplow-test/pop-event-data-and-user-id!)token-events (filter token-usage-event? events)simple-events (filter simple-event? events)]- (testing "no token_usage event on failure (chat-completion call failed)"- (is (empty? token-events)))- (testing "simple_event with failure result"+ (is (= "SELECT * FROM users" (:sql response)))+ (testing "token_usage event"+ (is (=? [{:data {"model_id" "claude-sonnet-4-5-20250929"+ "prompt_tokens" 1000+ "completion_tokens" 200+ "total_tokens" 1200+ "estimated_costs_usd" 0.0+ "duration_ms" 500+ "source" "oss_metabot"+ "tag" "oss-sqlgen"}}]+ token-events)))+ (testing "simple_event"(is (=? [{:data {"event" "metabot_oss_sqlgen_used""duration_ms" int?- "result" "failure"+ "result" "success""event_detail" "postgres"}}]simple-events))))))))))++(deftest generate-sql-snowplow-failure-test+ (testing "failed /generate-sql call tracks simple_event with failure result"+ (mt/with-temp [:model/Database db {:engine :postgres}+ :model/Table table {:db_id (:id db) :name "users" :schema "public"}+ :model/Field _ {:table_id (:id table) :name "id" :base_type :type/Integer}]+ (snowplow-test/with-fake-snowplow-collector+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly "sk-ant-test")+ llm.anthropic/chat-completion (fn [_] (throw (Exception. "API error")))]+ (mt/user-http-request :rasta :post 500 "llm/generate-sql"+ {:prompt "get all users"+ :database_id (:id db)+ :referenced_entities [{:model "table" :id (:id table)}]})+ (let [events (snowplow-test/pop-event-data-and-user-id!)+ token-events (filter token-usage-event? events)+ simple-events (filter simple-event? events)]+ (testing "no token_usage event on failure (chat-completion call failed)"+ (is (empty? token-events)))+ (testing "simple_event with failure result"+ (is (=? [{:data {"event" "metabot_oss_sqlgen_used"+ "duration_ms" int?+ "result" "failure"+ "event_detail" "postgres"}}]+ simple-events)))))))))diff --git a/test/metabase/llm/settings_test.clj b/test/metabase/llm/settings_test.cljindex 4e443b7b244..292759f4d83 100644--- a/test/metabase/llm/settings_test.clj+++ b/test/metabase/llm/settings_test.clj@@ -10,9 +10,10 @@(deftest llm-anthropic-api-key-setter-test(testing "accepts valid sk-ant- key and trims whitespace"- (mt/discard-setting-changes [llm-anthropic-api-key]- (llm.settings/llm-anthropic-api-key! " sk-ant-abc123 ")- (is (= "sk-ant-abc123" (llm.settings/llm-anthropic-api-key)))))+ (mt/with-temp-env-var-value! [mb-llm-anthropic-api-key nil]+ (mt/discard-setting-changes [llm-anthropic-api-key]+ (llm.settings/llm-anthropic-api-key! " sk-ant-abc123 ")+ (is (= "sk-ant-abc123" (llm.settings/llm-anthropic-api-key))))))(testing "rejects keys without sk-ant- prefix"(is (thrown-with-msg?@@ -21,22 +22,41 @@(llm.settings/llm-anthropic-api-key! "invalid-key"))))(testing "empty/nil clears the setting"- (mt/discard-setting-changes [llm-anthropic-api-key]- (llm.settings/llm-anthropic-api-key! "sk-ant-abc123")- (llm.settings/llm-anthropic-api-key! "")- (is (nil? (llm.settings/llm-anthropic-api-key))))))+ (mt/with-temp-env-var-value! [mb-llm-anthropic-api-key nil]+ (mt/discard-setting-changes [llm-anthropic-api-key]+ (llm.settings/llm-anthropic-api-key! "sk-ant-abc123")+ (llm.settings/llm-anthropic-api-key! "")+ (is (nil? (llm.settings/llm-anthropic-api-key)))))));;; ------------------------------------------- llm-anthropic-api-key-configured? Tests -------------------------------------------(deftest llm-anthropic-api-key-configured?-test(testing "returns false when no API key is set"- (mt/with-temporary-setting-values [llm-anthropic-api-key nil]+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)](is (false? (llm.settings/llm-anthropic-api-key-configured?)))))(testing "returns true when API key is set"(mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"](is (true? (llm.settings/llm-anthropic-api-key-configured?))))))+;;; ------------------------------------------- llm-proxy-base-url Feature Guard Tests -------------------------------------------++(deftest llm-proxy-base-url-feature-guard-test+ (testing "can be set and read when :metabase-ai-provider feature is enabled"+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-temporary-setting-values [llm-proxy-base-url "https://proxy.example"]+ (is (= "https://proxy.example" (llm.settings/llm-proxy-base-url)))+ (testing "returns default (nil) when :metabase-ai-provider feature is not enabled, even if a value is set"+ (mt/with-premium-features #{}+ (is (nil? (llm.settings/llm-proxy-base-url))))))))++ (testing "cannot be set when :metabase-ai-provider feature is not enabled"+ (mt/with-premium-features #{}+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Setting llm-proxy-base-url is not enabled because feature :metabase-ai-provider is not available"+ (llm.settings/llm-proxy-base-url! "https://proxy.example"))))))+;;; ------------------------------------------- Settings Defaults Tests -------------------------------------------(deftest llm-max-tokens-testdiff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 91421da078d..43f90e0ec45 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -4,6 +4,7 @@[metabase.analytics.prometheus :as prometheus][metabase.analytics.snowplow-test :as snowplow-test][metabase.lib.core :as lib]+ [metabase.llm.settings :as llm.settings][metabase.metabot.agent.analytics :as agent-analytics][metabase.metabot.agent.core :as agent][metabase.metabot.agent.memory :as memory]@@ -438,12 +439,13 @@:profile-id :embedding_next:context {}})))usages (filterv #(= :usage (:type %)) result)]- (testing "different models accumulate independently"- (is (= "model-a" (:model (first usages))))+ (testing "model is always the canonical provider-and-model from the profile"+ (is (= test-provider (:model (first usages))))+ (is (= test-provider (:model (second usages)))))+ (testing "usage accumulates under the single provider key"(is (= {:promptTokens 100 :completionTokens 20}(:usage (first usages))))- (is (= "model-b" (:model (second usages))))- (is (= {:promptTokens 200 :completionTokens 40}+ (is (= {:promptTokens 300 :completionTokens 60}(:usage (second usages))))))))))))(deftest run-agent-loop-retries-on-rate-limit-test@@ -753,11 +755,11 @@(run-agent-loop! opts)(is (zero? @classify-called))))(testing "does not call classify-and-track-user-intent-async! when api key is not configured"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"- llm-anthropic-api-key nil]- (run-agent-loop! opts)- (is (zero? @classify-called))))+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true+ llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]+ (run-agent-loop! opts)+ (is (zero? @classify-called)))))(testing "calls classify-and-track-user-intent-async! when llm-metabot-internal-tasks-enabled? is true"(mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? truellm-metabot-provider-lite "anthropic/claude-haiku-4-5"diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 1062e014e02..139e3c1e859 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -5,6 +5,7 @@[clojure.test :refer :all][compojure.response][medley.core :as m]+ [metabase.api.common :as mb.api][metabase.config.core :as config][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib]@@ -290,31 +291,32 @@(metabot.settings/llm-metabot-provider))))))(deftest settings-put-verifies-and-saves-api-keys-test- (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key nil]- (let [calls (atom 0)]- (with-redefs [metabot.self/list-models (fn [provider {:keys [api-key]}]- (swap! calls inc)- (is (= "anthropic" provider))- (is (= "sk-ant-valid" api-key))- (case @calls- 1 (is (nil? (llm.settings/llm-anthropic-api-key))- "verification should happen before saving the key")- 2 (is (= "sk-ant-valid" (llm.settings/llm-anthropic-api-key))- "response should use the saved key")- (is false (str "unexpected list-models call: " @calls)))- {:models [{:id "claude-haiku-4-5"- :display_name "Claude Haiku 4.5"}]})]- (is (= {:value (metabot.settings/llm-metabot-provider)- :models [{:id "claude-haiku-4-5"- :display_name "Claude Haiku 4.5"- :group "Haiku"}]}- (mt/user-http-request :crowberto :put 200 "metabot/settings"- {:provider "anthropic"- :api-key "sk-ant-valid"})))- (is (= 2 @calls)- "should verify first, then fetch models again after saving")- (is (= "sk-ant-valid"- (llm.settings/llm-anthropic-api-key)))))))+ (mt/with-temp-env-var-value! [mb-llm-anthropic-api-key nil]+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key nil]+ (let [calls (atom 0)]+ (with-redefs [metabot.self/list-models (fn [provider {:keys [api-key]}]+ (swap! calls inc)+ (is (= "anthropic" provider))+ (is (= "sk-ant-valid" api-key))+ (case (long @calls)+ 1 (is (nil? (llm.settings/llm-anthropic-api-key))+ "verification should happen before saving the key")+ 2 (is (= "sk-ant-valid" (llm.settings/llm-anthropic-api-key))+ "response should use the saved key")+ (is false (str "unexpected list-models call: " @calls)))+ {:models [{:id "claude-haiku-4-5"+ :display_name "Claude Haiku 4.5"}]})]+ (is (= {:value (metabot.settings/llm-metabot-provider)+ :models [{:id "claude-haiku-4-5"+ :display_name "Claude Haiku 4.5"+ :group "Haiku"}]}+ (mt/user-http-request :crowberto :put 200 "metabot/settings"+ {:provider "anthropic"+ :api-key "sk-ant-valid"})))+ (is (= 2 @calls)+ "should verify first, then fetch models again after saving")+ (is (= "sk-ant-valid"+ (llm.settings/llm-anthropic-api-key))))))))(deftest settings-put-rejects-invalid-api-key-test(mt/with-temporary-setting-values [llm.settings/llm-openai-api-key nil]@@ -498,6 +500,40 @@(into [] (#'api/combine-text-parts-xf)[{:type :text, :text "solo"}])))))+(defn- store-and-check!+ "Helper: call store-native-parts! with the given provider setting, return the stored message."+ [provider]+ (binding [mb.api/*current-user-id* (mt/user->id :crowberto)]+ (let [conv-id (str (random-uuid))]+ (try+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider provider]+ (#'api/store-native-parts!+ conv-id "internal"+ [{:type :start :id "msg-1"}+ {:type :text :text "Hello"}+ ;; SSE usage parts carry bare model names (from provider API response)+ {:type :usage :model "claude-sonnet-4-6" :usage {:promptTokens 100 :completionTokens 50}}+ {:type :data :data-type "state" :data {:step 1}}+ {:type :finish}])+ (t2/select-one :model/MetabotMessage :conversation_id conv-id))+ (finally+ (t2/delete! :model/MetabotMessage :conversation_id conv-id)+ (t2/delete! :model/MetabotConversation :id conv-id))))))++(deftest store-native-parts-ai-proxy-test+ (testing "metabase/ provider prefix sets ai_proxied true and stores bare model names"+ (let [msg (store-and-check! "metabase/anthropic/claude-sonnet-4-6")]+ (is (true? (:ai_proxied msg)))+ (is (= {:claude-sonnet-4-6 {:prompt 100 :completion 50}}+ (:usage msg))+ "usage keys should be bare model names, not metabase/anthropic/...")))++ (testing "BYOK provider (no metabase/ prefix) sets ai_proxied false"+ (let [msg (store-and-check! "anthropic/claude-sonnet-4-6")]+ (is (false? (:ai_proxied msg)))+ (is (= {:claude-sonnet-4-6 {:prompt 100 :completion 50}}+ (:usage msg))))))+(defn- legacy-query"A legacy inner-query-style map suitable for [[#'api/upgrade-viewing-queries]]."[]diff --git a/test/metabase/metabot/provider_util_test.clj b/test/metabase/metabot/provider_util_test.cljnew file mode 100644index 00000000000..27c3f0dae38--- /dev/null+++ b/test/metabase/metabot/provider_util_test.clj@@ -0,0 +1,59 @@+(ns metabase.metabot.provider-util-test+ (:require+ [clojure.test :refer :all]+ [metabase.metabot.provider-util :as provider-util]))++(deftest ^:parallel metabase-provider?-test+ (testing "returns true for metabase/ prefixed providers"+ (is (true? (provider-util/metabase-provider? "metabase/anthropic/claude-haiku-4-5")))+ (is (true? (provider-util/metabase-provider? "metabase/openai/gpt-4.1-mini")))+ (is (true? (provider-util/metabase-provider? "metabase/openrouter/anthropic/claude-haiku-4-5"))))+ (testing "returns false for direct providers"+ (is (false? (provider-util/metabase-provider? "anthropic/claude-haiku-4-5")))+ (is (false? (provider-util/metabase-provider? "openrouter/anthropic/claude-haiku-4-5"))))+ (testing "returns false for nil and empty string"+ (is (false? (provider-util/metabase-provider? nil)))+ (is (false? (provider-util/metabase-provider? "")))))++(deftest ^:parallel provider-and-model->outer-provider-test+ (testing "returns top-level prefix"+ (is (= "anthropic" (provider-util/provider-and-model->outer-provider "anthropic/claude-haiku-4-5")))+ (is (= "openrouter" (provider-util/provider-and-model->outer-provider "openrouter/anthropic/claude-haiku-4-5")))+ (is (= "metabase" (provider-util/provider-and-model->outer-provider "metabase/anthropic/claude-haiku-4-5"))))+ (testing "returns nil for nil and empty string"+ (is (nil? (provider-util/provider-and-model->outer-provider nil)))+ (is (= "" (provider-util/provider-and-model->outer-provider "")))))++(deftest ^:parallel provider-and-model->provider-test+ (testing "direct providers"+ (is (= "anthropic" (provider-util/provider-and-model->provider "anthropic/claude-haiku-4-5")))+ (is (= "openai" (provider-util/provider-and-model->provider "openai/gpt-4.1-mini")))+ (is (= "openrouter" (provider-util/provider-and-model->provider "openrouter/anthropic/claude-haiku-4-5"))))+ (testing "metabase/ prefix extracts inner provider"+ (is (= "anthropic" (provider-util/provider-and-model->provider "metabase/anthropic/claude-haiku-4-5")))+ (is (= "openai" (provider-util/provider-and-model->provider "metabase/openai/gpt-4.1-mini")))+ (is (= "openrouter" (provider-util/provider-and-model->provider "metabase/openrouter/anthropic/claude-haiku-4-5"))))+ (testing "returns nil for nil and empty string"+ (is (nil? (provider-util/provider-and-model->provider nil)))+ (is (= "" (provider-util/provider-and-model->provider "")))))++(deftest ^:parallel provider-and-model->model-test+ (testing "direct providers"+ (is (= "claude-haiku-4-5" (provider-util/provider-and-model->model "anthropic/claude-haiku-4-5")))+ (is (= "gpt-4.1-mini" (provider-util/provider-and-model->model "openai/gpt-4.1-mini")))+ (is (= "anthropic/claude-haiku-4-5" (provider-util/provider-and-model->model "openrouter/anthropic/claude-haiku-4-5"))))+ (testing "metabase/ prefix extracts model past inner provider"+ (is (= "claude-haiku-4-5" (provider-util/provider-and-model->model "metabase/anthropic/claude-haiku-4-5")))+ (is (= "gpt-4.1-mini" (provider-util/provider-and-model->model "metabase/openai/gpt-4.1-mini")))+ (is (= "anthropic/claude-haiku-4-5" (provider-util/provider-and-model->model "metabase/openrouter/anthropic/claude-haiku-4-5"))))+ (testing "returns nil for nil and empty string"+ (is (nil? (provider-util/provider-and-model->model nil)))+ (is (nil? (provider-util/provider-and-model->model "")))))++(deftest ^:parallel strip-metabase-prefix-test+ (testing "strips metabase/ prefix"+ (is (= "anthropic/claude-haiku-4-5" (provider-util/strip-metabase-prefix "metabase/anthropic/claude-haiku-4-5")))+ (is (= "openrouter/anthropic/claude-haiku-4-5" (provider-util/strip-metabase-prefix "metabase/openrouter/anthropic/claude-haiku-4-5"))))+ (testing "returns input unchanged when no prefix"+ (is (= "anthropic/claude-haiku-4-5" (provider-util/strip-metabase-prefix "anthropic/claude-haiku-4-5")))+ (is (= "openrouter/anthropic/claude-haiku-4-5" (provider-util/strip-metabase-prefix "openrouter/anthropic/claude-haiku-4-5")))))diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex 5cfac52eae1..ccb3b2952a4 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -1,17 +1,21 @@(ns metabase.metabot.self.claude-test(:require+ [clj-http.client :as http][clojure.test :refer :all][medley.core :as m]+ [metabase.llm.settings :as llm.settings][metabase.metabot.self.claude :as claude][metabase.metabot.self.core :as self.core]- [metabase.metabot.test-util :as test-util]))+ [metabase.metabot.test-util :as metabot.tu]+ [metabase.premium-features.core :as premium-features]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)(defn- fixture"Load cached Claude raw chunks, or capture from the API when `*live*` / no cache."[fixture-name opts]- (test-util/raw-fixture fixture-name #(claude/claude-raw (merge {:model "claude-haiku-4-5"} opts))))+ (metabot.tu/raw-fixture fixture-name #(claude/claude-raw (merge {:model "claude-haiku-4-5"} opts))));;; ──────────────────────────────────────────────────────────────────;;; Streaming chunk conversion tests@@ -32,7 +36,7 @@(deftest ^:parallel claude-tool-input-conv-test(let [raw-chunks (fixture "claude-tool-input"{:input [{:role :user :content "What time is it in Kyiv?"}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "tool input chunks are mapped correctly"(is (=? [{:type :start} {:type :tool-input-start} {:type :tool-input-delta} {:type :tool-input-available} {:type :usage}](into [] (comp (claude/claude->aisdk-chunks-xf) (m/distinct-by :type)) raw-chunks))))@@ -45,7 +49,7 @@(deftest ^:parallel claude-text-and-tool-input-conv-test(let [raw-chunks (fixture "claude-text-and-tool-input"{:input [{:role :user :content "Tell me what time it is in Kyiv. First explain what you're going to do, then call the tool."}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "text + tool input chunks are mapped correctly"(is (=? [{:type :start}{:type :text-start} {:type :text-delta} {:type :text-end}@@ -62,7 +66,7 @@(deftest ^:parallel claude-lite-aisdk-xf-test(let [raw-chunks (fixture "claude-text-and-tool-input"{:input [{:role :user :content "Tell me what time it is in Kyiv. First explain what you're going to do, then call the tool."}]- :tools [(test-util/get-time-tool)]})+ :tools [(metabot.tu/get-time-tool)]})res (into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/lite-aisdk-xf)) raw-chunks)](testing "lite-aisdk-xf collects tool inputs"(is (=? [{:type :start}@@ -158,3 +162,86 @@[{:type :text :text ""}{:type :text :text "Hello"}{:type :text :text ""}])))))++(deftest claude-auth-preferences-test+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-dynamic-fn-redefs [premium-features/premium-embedding-token (constantly "proxy-token")]+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-byok"+ llm.settings/llm-proxy-base-url "https://proxy.example"]+ (testing "Prefers BYOK over ai proxy"+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://api.anthropic.com/v1/messages"+ :headers {"x-api-key" "sk-ant-byok"}+ :body string?}+ (claude/claude-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Uses ai proxy when explicitly requested"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)+ self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://proxy.example/anthropic/v1/messages"+ :headers {"x-metabase-instance-token" "proxy-token"}+ :body string?}+ (claude/claude-raw {:input [{:role :user :content "hi"}]+ :ai-proxy? true})))))++ (testing "Does not fall back to ai proxy when BYOK is missing"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No Anthropic API key is set"+ (claude/claude-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Throws an error if nothing is defined"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm.settings/llm-proxy-base-url nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No Anthropic API key is set"+ (claude/claude-raw {:input [{:role :user :content "hi"}]}))))))))))++(deftest claude-list-models-auth-preferences-test+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-dynamic-fn-redefs [premium-features/premium-embedding-token (constantly "proxy-token")]+ (mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-byok"+ llm.settings/llm-proxy-base-url "https://proxy.example"]+ (testing "Prefers BYOK over ai proxy"+ (with-redefs [http/request (fn [req]+ (is (=? {:method :get+ :url "https://api.anthropic.com/v1/models"+ :headers {"anthropic-version" "2023-06-01"+ "x-api-key" "sk-ant-byok"}}+ req))+ {:body "{\"data\":[]}"})]+ (is (= {:models []}+ (claude/list-models {})))))++ (testing "Uses ai proxy when explicitly requested"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)+ http/request (fn [req]+ (is (=? {:method :get+ :url "https://proxy.example/anthropic/v1/models"+ :headers {"anthropic-version" "2023-06-01"+ "x-metabase-instance-token" "proxy-token"}}+ req))+ {:body "{\"data\":[]}"})]+ (is (= {:models []}+ (claude/list-models {:ai-proxy? true})))))++ (testing "Does not fall back to ai proxy when BYOK is missing"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No Anthropic API key is set"+ (claude/list-models {})))))++ (testing "Throws an error if nothing is defined"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm.settings/llm-proxy-base-url nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No Anthropic API key is set"+ (claude/list-models {}))))))))))diff --git a/test/metabase/metabot/self/openai_test.clj b/test/metabase/metabot/self/openai_test.cljindex 4e52bc25073..7a63bfd644b 100644--- a/test/metabase/metabot/self/openai_test.clj+++ b/test/metabase/metabot/self/openai_test.clj@@ -1,17 +1,21 @@(ns metabase.metabot.self.openai-test(:require+ [clj-http.client :as http][clojure.test :refer :all][medley.core :as m]+ [metabase.llm.settings :as llm.settings][metabase.metabot.self.core :as self.core][metabase.metabot.self.openai :as openai]- [metabase.metabot.test-util :as test-util]))+ [metabase.metabot.test-util :as metabot.tu]+ [metabase.premium-features.core :as premium-features]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)(defn- fixture"Load cached OpenAI raw chunks, or capture from the API when `*live*` / no cache."[fixture-name opts]- (test-util/raw-fixture fixture-name #(openai/openai-raw (merge {:model "gpt-4.1-mini"} opts))))+ (metabot.tu/raw-fixture fixture-name #(openai/openai-raw (merge {:model "gpt-4.1-mini"} opts))));;; ──────────────────────────────────────────────────────────────────;;; Streaming chunk conversion tests@@ -26,13 +30,15 @@(testing "through full pipeline produces text + usage"(is (=? [{:type :start}{:type :text :text string?}- {:type :usage :usage map?}]+ {:type :usage+ :usage {:promptTokens pos-int?+ :completionTokens pos-int?}}](into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))(deftest ^:parallel openai-tool-calls-conv-test(let [raw-chunks (fixture "openai-tool-calls"{:input [{:role :user :content "What time is it in Kyiv?"}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "tool call chunks are mapped correctly"(is (=? [{:type :start} {:type :tool-input-start} {:type :tool-input-delta} {:type :tool-input-available} {:type :usage}](into [] (comp (openai/openai->aisdk-chunks-xf) (m/distinct-by :type)) raw-chunks))))@@ -41,7 +47,9 @@(is (=? [{:type :start}{:type :tool-input :function string? :arguments map?}](take 2 parts)))- (is (=? {:type :usage :usage map?}+ (is (=? {:type :usage+ :usage {:promptTokens pos-int?+ :completionTokens pos-int?}}(last parts)))))))(deftest ^:parallel openai-structured-output-conv-test@@ -58,13 +66,15 @@(testing "through full pipeline produces text + usage"(is (=? [{:type :start}{:type :text :text string?}- {:type :usage :usage map?}]+ {:type :usage+ :usage {:promptTokens pos-int?+ :completionTokens pos-int?}}](into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))(deftest ^:parallel openai-text-and-tool-calls-conv-test(let [raw-chunks (fixture "openai-text-and-tool-calls"{:input [{:role :user :content "Tell me what time it is in Kyiv. First explain what you're going to do, then call the tool."}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "text + tool call chunks contain expected types"(is (=? [{:type :start}{:type :text-start} {:type :text-delta} {:type :text-end}@@ -75,9 +85,37 @@(is (=? [{:type :start}{:type :text :text string?}{:type :tool-input :function "get-time" :arguments {:tz string?}}- {:type :usage :usage map?}]+ {:type :usage+ :usage {:promptTokens pos-int?+ :completionTokens pos-int?}}](into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))+;;; ──────────────────────────────────────────────────────────────────+;;; Usage normalization tests+;;; ──────────────────────────────────────────────────────────────────++(deftest ^:parallel openai-usage-strips-nested-details-test+ (testing "nested *_details maps from reasoning models are stripped"+ (let [raw-chunks (fixture "openai-text"+ {:input [{:role :user :content "Say hello briefly, in under 10 words."}]})+ ;; Patch the last chunk to include nested usage details like reasoning models return+ patched (mapv (fn [chunk]+ (if (= (:type chunk) "response.completed")+ (assoc-in chunk [:response :usage]+ {:input_tokens 20+ :output_tokens 8+ :total_tokens 28+ :input_tokens_details {:cached_tokens 5 :audio_tokens 0}+ :output_tokens_details {:reasoning_tokens 3 :audio_tokens 0}})+ chunk))+ raw-chunks)+ parts (into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) patched)+ usage (:usage (last parts))]+ ;; no nested maps — safe for merge-with + in accumulate-usage-xf+ (is (= {:promptTokens 20+ :completionTokens 8}+ usage)))))+;;; ──────────────────────────────────────────────────────────────────;;; parts->openai-input tests;;; ──────────────────────────────────────────────────────────────────@@ -125,3 +163,43 @@:output #"Error:.*failed"}](openai/parts->openai-input[{:type :tool-output :id "call-1" :error {:message "Tool failed"}}])))))++(deftest openai-auth-preferences-test+ (mt/with-premium-features #{:metabase-ai-provider}+ (with-redefs [premium-features/premium-embedding-token (constantly "proxy-token")]+ (mt/with-temporary-setting-values [llm.settings/llm-openai-api-key "sk-ant-byok"+ llm.settings/llm-proxy-base-url "https://proxy.example"]+ (testing "Prefers BYOK over ai proxy"+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://api.openai.com/v1/responses"+ :headers {"Authorization" "Bearer sk-ant-byok"}+ :body string?}+ (openai/openai-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Uses ai proxy when explicitly requested"+ (mt/with-temporary-setting-values [llm.settings/llm-openai-api-key nil]+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://proxy.example/openai/v1/responses"+ :headers {"x-metabase-instance-token" "proxy-token"}+ :body string?}+ (openai/openai-raw {:input [{:role :user :content "hi"}]+ :ai-proxy? true}))))))++ (testing "Does not fall back to ai proxy when BYOK is missing"+ (mt/with-temporary-setting-values [llm.settings/llm-openai-api-key nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No OpenAI API key is set"+ (openai/openai-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Throws an error if nothing is defined"+ (mt/with-temporary-setting-values [llm.settings/llm-openai-api-key nil+ llm.settings/llm-proxy-base-url nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No OpenAI API key is set"+ (openai/openai-raw {:input [{:role :user :content "hi"}]})))))))))diff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.cljindex 82eb1a8fbfc..ba5cfbef9d4 100644--- a/test/metabase/metabot/self/openrouter_test.clj+++ b/test/metabase/metabot/self/openrouter_test.clj@@ -1,17 +1,21 @@(ns metabase.metabot.self.openrouter-test(:require+ [clj-http.client :as http][clojure.test :refer :all][medley.core :as m]+ [metabase.llm.settings :as llm.settings][metabase.metabot.self.core :as self.core][metabase.metabot.self.openrouter :as openrouter]- [metabase.metabot.test-util :as test-util]))+ [metabase.metabot.test-util :as metabot.tu]+ [metabase.premium-features.core :as premium-features]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)(defn- fixture"Load cached OpenRouter raw chunks, or capture from the API when `*live*` / no cache."[fixture-name opts]- (test-util/raw-fixture fixture-name #(openrouter/openrouter-raw (merge {:model "anthropic/claude-haiku-4-5"} opts))))+ (metabot.tu/raw-fixture fixture-name #(openrouter/openrouter-raw (merge {:model "anthropic/claude-haiku-4-5"} opts))));;; ──────────────────────────────────────────────────────────────────;;; parts->cc-messages tests@@ -105,7 +109,7 @@(deftest ^:parallel openrouter-tool-calls-conv-test(let [raw-chunks (fixture "openrouter-tool-calls"{:input [{:role :user :content "What time is it in Kyiv?"}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "single tool call chunks are mapped correctly"(is (=? [{:type :start} {:type :tool-input-start} {:type :tool-input-delta} {:type :tool-input-available} {:type :usage}](into [] (comp (openrouter/openrouter->aisdk-chunks-xf) (m/distinct-by :type)) raw-chunks))))@@ -123,8 +127,8 @@(deftest ^:parallel openrouter-parallel-tool-calls-conv-test(let [raw-chunks (fixture "openrouter-parallel-tool-calls"{:input [{:role :user :content "What time is it in Kyiv AND convert 100 EUR to USD? Use both tools in parallel."}]- :tools [(test-util/get-time-tool)- (test-util/convert-currency-tool)]})]+ :tools [(metabot.tu/get-time-tool)+ (metabot.tu/convert-currency-tool)]})](testing "parallel tool calls are mapped correctly"(is (=? [{:type :start} {:type :tool-input-start} {:type :tool-input-delta} {:type :tool-input-available} {:type :usage}](into [] (comp (openrouter/openrouter->aisdk-chunks-xf) (m/distinct-by :type)) raw-chunks))))@@ -144,7 +148,7 @@(deftest ^:parallel openrouter-text-and-tool-calls-conv-test(let [raw-chunks (fixture "openrouter-text-and-tool-calls"{:input [{:role :user :content "Tell me what time it is in Kyiv. First explain what you're going to do, then call the tool."}]- :tools [(test-util/get-time-tool)]})]+ :tools [(metabot.tu/get-time-tool)]})](testing "text + tool call chunks contain expected types"(is (=? [{:type :start}{:type :text-start} {:type :text-delta} {:type :text-end}@@ -167,7 +171,7 @@(testing "lite-aisdk-xf streams text deltas for openrouter format"(let [raw-chunks (fixture "openrouter-text-and-tool-calls"{:input [{:role :user :content "Tell me what time it is in Kyiv. First explain what you're going to do, then call the tool."}]- :tools [(test-util/get-time-tool)]})+ :tools [(metabot.tu/get-time-tool)]})res (into [] (comp (openrouter/openrouter->aisdk-chunks-xf)(self.core/lite-aisdk-xf))raw-chunks)]@@ -179,3 +183,43 @@:usage {:promptTokens pos-int? :completionTokens pos-int?}}](remove #(= (:type %) :text) res)))(is (< 10 (count (filter #(= (:type %) :text) res)))))))++(deftest openrouter-auth-preferences-test+ (mt/with-premium-features #{:metabase-ai-provider}+ (with-redefs [premium-features/premium-embedding-token (constantly "proxy-token")]+ (mt/with-temporary-setting-values [llm.settings/llm-openrouter-api-key "sk-or-v1-byok"+ llm.settings/llm-proxy-base-url "https://proxy.example"]+ (testing "Prefers BYOK over ai proxy"+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://openrouter.ai/api/v1/chat/completions"+ :headers {"Authorization" "Bearer sk-or-v1-byok"}+ :body string?}+ (openrouter/openrouter-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Uses ai proxy when explicitly requested"+ (mt/with-temporary-setting-values [llm.settings/llm-openrouter-api-key nil]+ (with-redefs [self.core/sse-reducible identity+ http/request (fn [req] {:body req})]+ (is (=? {:method :post+ :url "https://proxy.example/openrouter/v1/chat/completions"+ :headers {"x-metabase-instance-token" "proxy-token"}+ :body string?}+ (openrouter/openrouter-raw {:input [{:role :user :content "hi"}]+ :ai-proxy? true}))))))++ (testing "Does not fall back to ai proxy when BYOK is missing"+ (mt/with-temporary-setting-values [llm.settings/llm-openrouter-api-key nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No OpenRouter API key is set"+ (openrouter/openrouter-raw {:input [{:role :user :content "hi"}]})))))++ (testing "Throws an error if nothing is defined"+ (mt/with-temporary-setting-values [llm.settings/llm-openrouter-api-key nil+ llm.settings/llm-proxy-base-url nil]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No OpenRouter API key is set"+ (openrouter/openrouter-raw {:input [{:role :user :content "hi"}]})))))))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 06ed431b004..36443b49aaf 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -20,14 +20,21 @@(deftest ^:parallel parse-provider-model-test(testing "parses provider/model format correctly"- (is (=? {:provider "anthropic" :model "claude-haiku-4-5"}+ (is (=? {:provider "anthropic" :model "claude-haiku-4-5" :ai-proxy? false}(#'self/parse-provider-model "anthropic/claude-haiku-4-5")))- (is (=? {:provider "openai" :model "gpt-4.1-mini"}+ (is (=? {:provider "openai" :model "gpt-4.1-mini" :ai-proxy? false}(#'self/parse-provider-model "openai/gpt-4.1-mini")))- (is (=? {:provider "openrouter" :model "anthropic/claude-haiku-4-5"}+ (is (=? {:provider "openrouter" :model "anthropic/claude-haiku-4-5" :ai-proxy? false}(#'self/parse-provider-model "openrouter/anthropic/claude-haiku-4-5")))- (is (=? {:provider "openrouter" :model "google/gemini-2.5-flash"}+ (is (=? {:provider "openrouter" :model "google/gemini-2.5-flash" :ai-proxy? false}(#'self/parse-provider-model "openrouter/google/gemini-2.5-flash"))))+ (testing "parses metabase/ prefix (AI proxy)"+ (is (=? {:provider "anthropic" :model "claude-haiku-4-5" :ai-proxy? true}+ (#'self/parse-provider-model "metabase/anthropic/claude-haiku-4-5")))+ (is (=? {:provider "openai" :model "gpt-4.1-mini" :ai-proxy? true}+ (#'self/parse-provider-model "metabase/openai/gpt-4.1-mini")))+ (is (=? {:provider "openrouter" :model "anthropic/claude-haiku-4-5" :ai-proxy? true}+ (#'self/parse-provider-model "metabase/openrouter/anthropic/claude-haiku-4-5"))))(testing "throws for invalid formats/models"(is (thrown-with-msg? Exception #"Unknown LLM provider: no-slash" (#'self/parse-provider-model "no-slash")))(is (thrown-with-msg? Exception #"Unknown LLM provider: " (#'self/parse-provider-model "")))@@ -45,9 +52,10 @@(deftest call-llm-tool-choice-test(testing "passes required tool choice to LLM providers"(let [captured (atom nil)]- (mt/with-dynamic-fn-redefs [http/post (fn [_url opts]- (reset! captured (json/decode+kw (:body opts)))- (throw (ex-info "stop" {::skip true :status 401 :body "skip parsing"})))]+ (mt/with-dynamic-fn-redefs [http/request (fn [opts]+ (when (:body opts)+ (reset! captured (json/decode+kw (:body opts))))+ (throw (ex-info "stop" {::skip true :status 401 :body "skip parsing"})))](mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"llm-openrouter-api-key "sk-or-v1-test-key"llm-openai-api-key "sk-test-key"]diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.cljindex 7734fd38512..76605782d8b 100644--- a/test/metabase/metabot/settings_test.clj+++ b/test/metabase/metabot/settings_test.clj@@ -1,6 +1,7 @@(ns metabase.metabot.settings-test(:require[clojure.test :refer [deftest is testing use-fixtures]]+ [metabase.llm.settings :as llm.settings][metabase.metabot.settings :as metabot.settings][metabase.test :as mt][metabase.test.fixtures :as fixtures]))@@ -76,6 +77,94 @@(is (string? (metabot.settings/llm-metabot-provider)))(is (string? (metabot.settings/llm-metabot-provider-lite))))))+(deftest metabot-configured-with-metabase-provider-and-proxy-url-test+ (testing "returns true when metabase-proxied provider and proxy URL is set"+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4"+ llm-proxy-base-url "https://proxy.example.com"]+ (is (true? (metabot.settings/llm-metabot-configured?)))))))++(deftest metabot-configured-with-metabase-provider-no-proxy-url-test+ (testing "returns false when metabase-proxied provider and no proxy URL"+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4"+ llm-proxy-base-url nil]+ (is (false? (metabot.settings/llm-metabot-configured?)))))))++(deftest metabot-configured-with-direct-provider-and-api-key-test+ (testing "returns true when direct provider has API key set"+ (mt/with-temporary-setting-values [llm-metabot-provider "anthropic/claude-sonnet-4"+ llm-anthropic-api-key "sk-ant-test"]+ (is (true? (metabot.settings/llm-metabot-configured?))))))++(deftest metabot-configured-with-direct-provider-no-api-key-test+ (testing "returns false when direct provider has no API key"+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm-metabot-provider "anthropic/claude-sonnet-4"]+ (is (false? (metabot.settings/llm-metabot-configured?)))))))++(deftest metabot-configured-proxy-url-not-fallback-for-direct-provider-test+ (testing "proxy URL alone does not make a direct provider configured"+ (mt/with-premium-features #{:metabase-ai-provider}+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm-metabot-provider "anthropic/claude-sonnet-4"+ llm-proxy-base-url "https://proxy.example.com"]+ (is (false? (metabot.settings/llm-metabot-configured?))))))))++;;; ------------------------------------------- validate-metabot-provider! Tests -------------------------------------------+;; The validator is private; exercise it through the setting setter.++(deftest validate-metabot-provider-rejects-non-string-test+ (testing "rejects non-string input"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"must be a string"+ (metabot.settings/llm-metabot-provider! 42)))))++(deftest validate-metabot-provider-rejects-unknown-provider-test+ (testing "rejects unknown top-level provider"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"Unknown provider"+ (metabot.settings/llm-metabot-provider! "foobar/some-model")))))++(deftest validate-metabot-provider-rejects-blank-model-test+ (testing "rejects provider-only string with no model"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"Model name is required"+ (metabot.settings/llm-metabot-provider! "anthropic/")))))++(deftest validate-metabot-provider-rejects-unknown-inner-provider-test+ (testing "rejects unknown inner provider under metabase/ prefix"+ (mt/with-premium-features #{:metabase-ai-provider}+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"Unknown inner provider"+ (metabot.settings/llm-metabot-provider! "metabase/foobar/some-model"))))))++(deftest validate-metabot-provider-rejects-metabase-prefix-without-model-test+ (testing "rejects metabase/provider with no model"+ (mt/with-premium-features #{:metabase-ai-provider}+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"Model name is required"+ (metabot.settings/llm-metabot-provider! "metabase/anthropic/"))))))++(deftest validate-metabot-provider-accepts-valid-direct-providers-test+ (testing "accepts valid direct provider strings"+ (mt/with-temporary-setting-values [llm-metabot-provider "anthropic/claude-sonnet-4"]+ (is (= "anthropic/claude-sonnet-4" (metabot.settings/llm-metabot-provider))))+ (mt/with-temporary-setting-values [llm-metabot-provider "openai/gpt-4.1-mini"]+ (is (= "openai/gpt-4.1-mini" (metabot.settings/llm-metabot-provider))))+ (mt/with-temporary-setting-values [llm-metabot-provider "openrouter/anthropic/claude-haiku-4-5"]+ (is (= "openrouter/anthropic/claude-haiku-4-5" (metabot.settings/llm-metabot-provider))))))++(deftest validate-metabot-provider-accepts-valid-metabase-prefixed-test+ (testing "accepts valid metabase/ prefixed provider strings"+ (mt/with-premium-features #{:metabase-ai-provider}+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4"]+ (is (= "metabase/anthropic/claude-sonnet-4" (metabot.settings/llm-metabot-provider))))+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/openrouter/anthropic/claude-haiku-4-5"]+ (is (= "metabase/openrouter/anthropic/claude-haiku-4-5" (metabot.settings/llm-metabot-provider)))))))++;;; ------------------------------------------- Internal Tasks Tests -------------------------------------------+(deftest internal-tasks-disabled-by-default-test(testing "returns false when the setting is not explicitly enabled"(mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? nil@@ -84,10 +173,10 @@(deftest internal-tasks-requires-lite-api-key-test(testing "returns false when enabled but lite provider has no api key"- (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true- llm-metabot-provider-lite "anthropic/claude-haiku-4-5"- llm-anthropic-api-key nil]- (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?))))))+ (with-redefs [llm.settings/llm-anthropic-api-key (constantly nil)]+ (mt/with-temporary-setting-values [llm-metabot-internal-tasks-enabled? true+ llm-metabot-provider-lite "anthropic/claude-haiku-4-5"]+ (is (false? (metabot.settings/llm-metabot-internal-tasks-enabled?)))))))(deftest internal-tasks-enabled-with-anthropic-test(testing "returns true when enabled and anthropic lite provider api key is configured"diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex 13ee19a2e57..ad4762fc034 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -1,6 +1,7 @@(ns metabase.slackbot.streaming-test(:require[clojure.test :refer :all]+ [metabase.metabot.persistence :as metabot.persistence][metabase.slackbot.client :as slackbot.client][metabase.slackbot.persistence :as slackbot.persistence][metabase.slackbot.streaming :as slackbot.streaming]@@ -148,6 +149,54 @@(is (= "metabot_feedback" (:block_id (first blocks))))(is (= "feedback_buttons" (:type (first (:elements (first blocks)))))))))))))))+(deftest slackbot-streaming-sets-ai-proxied-on-messages-test+ (testing "store-message! receives ai-proxy? = true for metabase/ prefixed provider"+ (tu/with-slackbot-setup+ (let [event-body tu/base-dm-event+ store-opts (atom [])]+ (tu/with-slackbot-mocks+ {:ai-text "Hello!"}+ (fn [{:keys [stop-stream-calls]}]+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-haiku-4-5"]+ (with-redefs [metabot.persistence/store-message!+ (fn [_conv-id _profile-id _messages & {:as opts}]+ (swap! store-opts conj opts)+ nil)]+ (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body)+ (u/poll {:thunk #(>= (count @stop-stream-calls) 1)+ :done? true?+ :timeout-ms 5000})))+ (testing "user + assistant store-message! calls both received ai-proxy? = true"+ (is (=? [{:ai-proxy? true}+ {:ai-proxy? true}]+ @store-opts)))))))))++(deftest slackbot-streaming-sets-ai-proxied-false-for-byok-test+ (testing "store-message! receives ai-proxy? = false for direct BYOK provider"+ (tu/with-slackbot-setup+ (let [event-body tu/base-dm-event+ store-opts (atom [])]+ (tu/with-slackbot-mocks+ {:ai-text "Hello!"}+ (fn [{:keys [stop-stream-calls]}]+ (mt/with-temporary-setting-values [llm-metabot-provider "anthropic/claude-haiku-4-5"]+ (with-redefs [metabot.persistence/store-message!+ (fn [_conv-id _profile-id _messages & {:as opts}]+ (swap! store-opts conj opts)+ nil)]+ (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body)+ (u/poll {:thunk #(>= (count @stop-stream-calls) 1)+ :done? true?+ :timeout-ms 5000})))+ (testing "user + assistant store-message! calls both received ai-proxy? = false"+ (is (=? [{:ai-proxy? false}+ {:ai-proxy? false}]+ @store-opts)))))))))+;;; ------------------------------------------------ Flush throttle tests ------------------------------------------------(defn- make-test-callbacks