The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Explain AI provider outages in Metabot instead of a generic error (#83854)
Andrei Simionescu6e70ede8f3c3
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.tsindex fd4779bf7fc..50a7eafa1eb 100644--- a/frontend/src/metabase/metabot/state/actions.ts+++ b/frontend/src/metabase/metabot/state/actions.ts@@ -724,6 +724,7 @@ export const sendAgentRequest = createAsyncThunk<"ai_provider_billing","ai_provider_rate_limit","ai_provider_auth",+ "ai_provider_unavailable",),message: P.string,},diff --git a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsxindex 1ae375743e3..d4641977299 100644--- a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx@@ -127,6 +127,7 @@ describe("metabot > errors", () => {"ai_provider_billing","ai_provider_rate_limit","ai_provider_auth",+ "ai_provider_unavailable",])("should show the backend message for %s errors", async (errorCode) => {setup();mockAgentEndpoint({diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 00b3b0c7af2..6ebdbef2eb3 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -543,7 +543,7 @@{:type :text, :id (str (random-uuid)), :text message})(defn- error-part [^Exception e]- {:type :error, :error (or (self/byok-provider-error e)+ {:type :error, :error (or (self/provider-error e){:message (.getMessage e), :type (str (type e)), :data (ex-data e)})})(defn- accumulate-usage-xfdiff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 2a52d534c9e..c9fa26bb30b 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -349,48 +349,72 @@#{"billing_error" "enforced_spend_limit_reached" "insufficient_quota" "credit_balance_exhausted""organization_spend_limit_exceeded" "project_spend_limit_exceeded" "organization_usage_limit_exceeded"})+(def ^:private provider-unavailable-error-types+ "Error types Anthropic uses for a failure on its side, in a 5xx response or in an error event streamed after a 200."+ #{"api_error" "timeout_error" "overloaded_error"})+(defn- provider-failure- "Classify a provider API error's ex-data as `:billing`, `:rate-limit` or `:auth`, or nil for any other failure."- [{:keys [status body]}]- (let [{error-type :type error-code :code :keys [message details]} (:error body)]- (cond- (or (= status 402)- (some provider-billing-error-codes [error-type error-code (:error_code details)])- ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error- (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))- :billing-- (= status 429)- :rate-limit-- (or (= status 401) (= "permission_error" error-type))- :auth)))--(defn byok-provider-error- "A user-facing `{:message :error-code}` for a provider failure that the customer can fix on their side, or nil.- Always nil on the managed provider, where these failures are Metabase's to fix. Only admins are told which- provider failed and where to fix it."- [e]- (let [{:keys [api-error provider] :as data} (ex-data e)]- (when-let [failure (and api-error- provider- (not (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider)))- (provider-failure data))]- (let [admin? api/*is-superuser?*- provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]- (case failure- :billing {:error-code "ai_provider_billing"- :message (if admin?- (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)- (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}- :rate-limit {:error-code "ai_provider_rate_limit"- :message (if admin?- (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)- (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}- :auth {:error-code "ai_provider_auth"- :message (if admin?- (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)- (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))})))))+ "Classify a provider failure as `:billing`, `:rate-limit`, `:auth` or `:unavailable`, or nil for any other failure.+ Reads its HTTP `status`, nil for an error streamed after a 200, and the `error` object the provider sent."+ [status {error-type :type error-code :code :keys [message details]}]+ (cond+ (or (= status 402)+ (some provider-billing-error-codes [error-type error-code (:error_code details)])+ ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error+ (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))+ :billing++ (= status 429)+ :rate-limit++ (or (= status 401) (= "permission_error" error-type))+ :auth++ (or (some-> status (>= 500)) (provider-unavailable-error-types error-type))+ :unavailable))++(defn provider-error+ "A user-facing `{:message :error-code}` for a provider failure we can explain better than the generic error, or nil.+ Takes a provider API exception, or the failure's `provider`, `status` and `error` as [[provider-failure]] reads them.++ On the managed provider only an outage gets one, since billing, rate limits and credentials there are Metabase's to+ fix. Only admins on their own key are told which provider failed and where to fix it."+ ([e]+ (let [{:keys [api-error provider status body]} (ex-data e)]+ (when (and api-error provider)+ (provider-error provider status (:error body)))))+ ([provider status error]+ (when-let [failure (provider-failure status error)]+ (let [managed? (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider))+ admin? (and api/*is-superuser?* (not managed?))+ provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]+ (when (or (= :unavailable failure) (not managed?))+ (case failure+ :billing {:error-code "ai_provider_billing"+ :message (if admin?+ (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)+ (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}+ :rate-limit {:error-code "ai_provider_rate_limit"+ :message (if admin?+ (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)+ (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}+ :auth {:error-code "ai_provider_auth"+ :message (if admin?+ (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)+ (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))}+ :unavailable {:error-code "ai_provider_unavailable"+ :message (if admin?+ (tru "{0} is having problems right now. Try again in a few minutes." provider-name)+ (tru "The AI provider is having problems right now. Please try again in a few minutes."))}))))))++(defn- provider-error-xf+ "Transducer that replaces the error on a streamed `:error` part with [[provider-error]]'s, when it has one."+ [provider]+ (map (fn [part]+ (if-let [error (when (= :error (:type part))+ (provider-error provider nil (:error part)))]+ (assoc part :error error)+ part))))(defn- missing-required-permission"Returns the metabot permission keyword that the current user is missing@@ -515,6 +539,7 @@(core/lite-aisdk-xf)(core/stamp-tool-titles-xf tools)(report-aisdk-errors-xf tracking-opts)+ (provider-error-xf provider)(report-token-usage-xf tracking-opts)(report-tool-usage-xf tracking-opts tools))(stream-fn streaming-opts)))]diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.cljindex ea2d4101d51..2905e12ea14 100644--- a/src/metabase/metabot/self/claude.clj+++ b/src/metabase/metabot/self/claude.clj@@ -193,7 +193,8 @@(= t "message_stop") identity;; catch errors if any(= t "error") (rf {:type :error- :errorText (:message error)}))))))))+ :errorText (:message error)+ :error error}))))))));;; AISDK parts → Claude messagesdiff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 945d465452b..d4753d0e0f2 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -421,7 +421,8 @@;; Provider adapters emit AI SDK v5 error chunks (`{:type :error :errorText "..."}`). The rest of the;; pipeline — `format-error-line` and `metabot.persistence`'s turn finalize — keys off an internal;; `{:error {:message ...}}` part instead, so translate here. Internally-generated error parts (usage- ;; limits, agent-loop exceptions) already carry an `:error` map; pass those through unchanged.+ ;; limits, agent-loop exceptions) and Claude's error chunks, which keep the provider's own error+ ;; object, already carry an `:error` map; pass those through unchanged.:error (if (:error chunk)chunk{:type :error :error {:message (:errorText chunk)}})diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.cljindex 2fe539988a1..ad4c60b889f 100644--- a/src/metabase/slackbot/streaming.clj+++ b/src/metabase/slackbot/streaming.clj@@ -246,7 +246,7 @@"You do not have permission to use the AI assistant."(#{"metabase_ai_managed_locked" "ai_usage_limit_reached"- "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"} code)+ "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth" "ai_provider_unavailable"} code)(:message error)(provider-config-error-codes code)diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex f0ffd1a2174..520953d46cd 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -1003,36 +1003,49 @@"Should have called LLM twice (1 failure + 1 success")))))))(deftest provider-failure-error-part-test- (let [credit-error (ex-info "Anthropic API error (HTTP 400)"- {:status 400- :body {:type "error"- :error {:type "invalid_request_error"- :message "Your credit balance is too low to access the Anthropic API."}}- :api-error true- :provider "anthropic"- :error-code :provider-api-error})- error-parts (fn [model-ref]+ (let [api-error (fn [status error]+ (ex-info (str "Anthropic API error (HTTP " status ")")+ {:status status+ :body {:type "error" :error error}+ :api-error true+ :provider "anthropic"+ :error-code :provider-api-error}))+ credit-error (api-error 400 {:type "invalid_request_error"+ :message "Your credit balance is too low to access the Anthropic API."})+ overloaded {:type "overloaded_error" :message "Overloaded"}+ error-parts (fn [model-ref claude-fn](mt/with-temporary-setting-values [llm-providers llm.tu/default-connectionsllm-metabot-provider model-ref]- (mt/with-dynamic-fn-redefs [claude/claude (fn [_] (throw credit-error))]- (mt/with-log-level [metabase.metabot.agent.core :fatal]+ (mt/with-dynamic-fn-redefs [claude/claude claude-fn+ self/retry-delay-ms (constantly 0)]+ (mt/with-log-level [metabase.metabot :fatal](filterv #(= :error (:type %))(agent/run-agent-loop {:messages [{:role :user :content "Hi"}]:state {}:profile-id :embedding_next- :context {}}))))))]+ :context {}}))))))+ throw-credit (fn [_] (throw credit-error))](testing "on the customer's own key, the error part says what to fix instead of passing the provider's text on"(is (=? [{:error {:error-code "ai_provider_billing" :message #"Anthropic rejected the request .*"}}]- (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6"))))+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6" throw-credit))))(is (=? [{:error {:error-code "ai_provider_billing" :message #"The AI provider rejected the request .*"}}](mt/with-current-user (mt/user->id :rasta)- (error-parts "anthropic/claude-sonnet-4-6")))))+ (error-parts "anthropic/claude-sonnet-4-6" throw-credit)))))(testing "on the managed provider, the error part keeps today's generic shape"(is (= [{:type :error:error {:message (ex-message credit-error):type (str (type credit-error)):data (ex-data credit-error)}}]- (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6")))))))+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6" throw-credit)))))+ (testing "an outage says to try again later on either connection, whether the request failed or the stream did"+ (doseq [claude-fn [(fn [_] (throw (api-error 529 overloaded)))+ (fn [_] (eduction (claude/claude->aisdk-chunks-xf)+ [{:type "message_start" :message {:id "msg_1"}}+ {:type "error" :error overloaded}]))]]+ (is (=? [{:error {:error-code "ai_provider_unavailable" :message #"Anthropic is having problems .*"}}]+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6" claude-fn))))+ (is (=? [{:error {:error-code "ai_provider_unavailable" :message #"The AI provider is having problems .*"}}]+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6" claude-fn))))))));;; ===================== Prometheus Metrics Tests =====================diff --git a/test/metabase/metabot/self/bedrock_test.clj b/test/metabase/metabot/self/bedrock_test.cljindex 858b240dc4a..f93759eb956 100644--- a/test/metabase/metabot/self/bedrock_test.clj+++ b/test/metabase/metabot/self/bedrock_test.clj@@ -698,7 +698,9 @@(chunk-message {:type "message_stop":amazon-bedrock-invocationMetrics {:inputTokenCount 10 :outputTokenCount 2}})])))))(testing "an exception partway through becomes Claude's error chunk"- (is (= [{:type :error :errorText "Model stream error"}]+ (is (= [{:type :error+ :errorText "Model stream error"+ :error {:type "modelStreamErrorException" :message "Model stream error"}}](filter (comp #{:error} :type)(runtime-chunks-for! [message-start stream-error]))))))diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex d02a9ff6cc7..2dba56661c4 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -88,7 +88,7 @@{:type "error" :error {:type "overloaded_error" :message "Overloaded"}}]parts (into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)err (m/find-first #(= :error (:type %)) parts)]- (is (=? {:message "Overloaded"} (:error err)))+ (is (=? {:type "overloaded_error" :message "Overloaded"} (:error err)))(is (= (self.core/format-sse-event {:type "error" :errorText "Overloaded"})(self.core/format-error-line err))))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex a0116dcb0fa..74a9feedd20 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -2601,46 +2601,50 @@:error {:type "invalid_request_error":message "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits."}})-(deftest byok-provider-error-test+(deftest provider-error-test(mt/with-temporary-setting-values [llm-providers llm.tu/default-connectionsllm-metabot-provider "anthropic/claude-sonnet-4-6"]- (testing "classifies the failures that the customer can fix on their side"+ (testing "classifies the failures that the customer can fix on their side, and the provider's outages"(are [code provider status body]- (= code (:error-code (mt/as-admin (self/byok-provider-error (provider-api-error! provider status body)))))- "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body- "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"- :message "You have reached your specified API usage limits."}}- "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}- "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"- :message "You have reached your API usage limits."- :details {:error_code "enforced_spend_limit_reached"}}}- "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"- :code "insufficient_quota"- :message "You exceeded your current quota."}}- "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}- "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}- "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}- "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}- "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"- :message "Your API key does not have permission to use the specified resource."}}- nil "openai" 403 {:error {:type "request_forbidden"- :code "unsupported_country_region_territory"- :message "Country, region, or territory not supported"}}- nil "anthropic" 400 {:error {:type "invalid_request_error"- :message "max_tokens: Input should be greater than 0"}}- nil "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}))+ (= code (:error-code (mt/as-admin (self/provider-error (provider-api-error! provider status body)))))+ "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body+ "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"+ :message "You have reached your specified API usage limits."}}+ "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}+ "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"+ :message "You have reached your API usage limits."+ :details {:error_code "enforced_spend_limit_reached"}}}+ "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"+ :code "insufficient_quota"+ :message "You exceeded your current quota."}}+ "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}+ "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}+ "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}+ "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}+ "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"+ :message "Your API key does not have permission to use the specified resource."}}+ "ai_provider_unavailable" "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}+ "ai_provider_unavailable" "anthropic" 500 {:error {:type "api_error" :message "Internal server error"}}+ "ai_provider_unavailable" "openai" 503 {:error {:message "The server is overloaded or not ready yet."}}+ nil "openai" 403 {:error {:type "request_forbidden"+ :code "unsupported_country_region_territory"+ :message "Country, region, or territory not supported"}}+ nil "anthropic" 400 {:error {:type "invalid_request_error"+ :message "max_tokens: Input should be greater than 0"}}))(testing "admins are told which provider failed, everyone else is not"- (doseq [status [402 429 401]]+ (doseq [status [402 429 401 529]](let [e (provider-api-error! "anthropic" status {})]- (is (str/includes? (:message (mt/as-admin (self/byok-provider-error e)))+ (is (str/includes? (:message (mt/as-admin (self/provider-error e)))"Anthropic"))(is (not (str/includes? (:message (mt/with-current-user (mt/user->id :rasta)- (self/byok-provider-error e)))+ (self/provider-error e)))"Anthropic"))))))- (testing "the managed provider keeps the generic error, since its failures are Metabase's to fix"+ (testing "on the managed provider only an outage is explained, and without naming the provider"(mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"](is (nil? (mt/as-admin- (self/byok-provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))))))+ (self/provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))+ (is (=? {:error-code "ai_provider_unavailable" :message #"The AI provider is having problems.*"}+ (mt/as-admin (self/provider-error (provider-api-error! "anthropic" 529 {})))))))))(deftest known-models-normalization-test(testing "adapters that key model id to a map are passed through"diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex 42cd3560e00..a3ed24f53b0 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -391,8 +391,8 @@:type "clojure.lang.ExceptionInfo":data {:status-code 400 :api-error true :error-code :llm-not-configured}}})](is (str/includes? text "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."))))- (testing "a provider failure the customer can fix keeps the message the agent loop wrote for it"- (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"]]+ (testing "a provider failure the agent loop explained keeps the message it wrote for it"+ (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth" "ai_provider_unavailable"]](let [text (dm-error-part-appended-text!{:type :error :error {:message "Ask your administrator to check the AI provider." :error-code code}})](is (str/includes? text "Ask your administrator to check the AI provider.") code))))