Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Explain AI provider outages in Metabot instead of a generic error (#83854)

Andrei Simionescu6e70ede8f3c3
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.ts
index fd4779bf7fc..50a7eafa1eb 100644
--- a/frontend/src/metabase/metabot/state/actions.ts
+++ b/frontend/src/metabase/metabot/state/actions.ts
@@ -724,6 +724,7 @@ export const sendAgentRequest = createAsyncThunk<
"ai_provider_billing",
"ai_provider_rate_limit",
"ai_provider_auth",
+ "ai_provider_unavailable",
),
message: P.string,
},
diff --git a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
index 1ae375743e3..d4641977299 100644
--- a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
@@ -127,6 +127,7 @@ describe("metabot > errors", () => {
"ai_provider_billing",
"ai_provider_rate_limit",
"ai_provider_auth",
+ "ai_provider_unavailable",
])("should show the backend message for %s errors", async (errorCode) => {
setup();
mockAgentEndpoint({
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index 00b3b0c7af2..6ebdbef2eb3 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -543,7 +543,7 @@
{:type :text, :id (str (random-uuid)), :text message})
(defn- error-part [^Exception e]
- {:type :error, :error (or (self/byok-provider-error e)
+ {:type :error, :error (or (self/provider-error e)
{:message (.getMessage e), :type (str (type e)), :data (ex-data e)})})
(defn- accumulate-usage-xf
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 2a52d534c9e..c9fa26bb30b 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -349,48 +349,72 @@
#{"billing_error" "enforced_spend_limit_reached" "insufficient_quota" "credit_balance_exhausted"
"organization_spend_limit_exceeded" "project_spend_limit_exceeded" "organization_usage_limit_exceeded"})
+(def ^:private provider-unavailable-error-types
+ "Error types Anthropic uses for a failure on its side, in a 5xx response or in an error event streamed after a 200."
+ #{"api_error" "timeout_error" "overloaded_error"})
+
(defn- provider-failure
- "Classify a provider API error's ex-data as `:billing`, `:rate-limit` or `:auth`, or nil for any other failure."
- [{:keys [status body]}]
- (let [{error-type :type error-code :code :keys [message details]} (:error body)]
- (cond
- (or (= status 402)
- (some provider-billing-error-codes [error-type error-code (:error_code details)])
- ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error
- (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))
- :billing
-
- (= status 429)
- :rate-limit
-
- (or (= status 401) (= "permission_error" error-type))
- :auth)))
-
-(defn byok-provider-error
- "A user-facing `{:message :error-code}` for a provider failure that the customer can fix on their side, or nil.
- Always nil on the managed provider, where these failures are Metabase's to fix. Only admins are told which
- provider failed and where to fix it."
- [e]
- (let [{:keys [api-error provider] :as data} (ex-data e)]
- (when-let [failure (and api-error
- provider
- (not (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider)))
- (provider-failure data))]
- (let [admin? api/*is-superuser?*
- provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]
- (case failure
- :billing {:error-code "ai_provider_billing"
- :message (if admin?
- (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)
- (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}
- :rate-limit {:error-code "ai_provider_rate_limit"
- :message (if admin?
- (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)
- (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}
- :auth {:error-code "ai_provider_auth"
- :message (if admin?
- (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)
- (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))})))))
+ "Classify a provider failure as `:billing`, `:rate-limit`, `:auth` or `:unavailable`, or nil for any other failure.
+ Reads its HTTP `status`, nil for an error streamed after a 200, and the `error` object the provider sent."
+ [status {error-type :type error-code :code :keys [message details]}]
+ (cond
+ (or (= status 402)
+ (some provider-billing-error-codes [error-type error-code (:error_code details)])
+ ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error
+ (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))
+ :billing
+
+ (= status 429)
+ :rate-limit
+
+ (or (= status 401) (= "permission_error" error-type))
+ :auth
+
+ (or (some-> status (>= 500)) (provider-unavailable-error-types error-type))
+ :unavailable))
+
+(defn provider-error
+ "A user-facing `{:message :error-code}` for a provider failure we can explain better than the generic error, or nil.
+ Takes a provider API exception, or the failure's `provider`, `status` and `error` as [[provider-failure]] reads them.
+
+ On the managed provider only an outage gets one, since billing, rate limits and credentials there are Metabase's to
+ fix. Only admins on their own key are told which provider failed and where to fix it."
+ ([e]
+ (let [{:keys [api-error provider status body]} (ex-data e)]
+ (when (and api-error provider)
+ (provider-error provider status (:error body)))))
+ ([provider status error]
+ (when-let [failure (provider-failure status error)]
+ (let [managed? (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider))
+ admin? (and api/*is-superuser?* (not managed?))
+ provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]
+ (when (or (= :unavailable failure) (not managed?))
+ (case failure
+ :billing {:error-code "ai_provider_billing"
+ :message (if admin?
+ (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)
+ (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}
+ :rate-limit {:error-code "ai_provider_rate_limit"
+ :message (if admin?
+ (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)
+ (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}
+ :auth {:error-code "ai_provider_auth"
+ :message (if admin?
+ (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)
+ (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))}
+ :unavailable {:error-code "ai_provider_unavailable"
+ :message (if admin?
+ (tru "{0} is having problems right now. Try again in a few minutes." provider-name)
+ (tru "The AI provider is having problems right now. Please try again in a few minutes."))}))))))
+
+(defn- provider-error-xf
+ "Transducer that replaces the error on a streamed `:error` part with [[provider-error]]'s, when it has one."
+ [provider]
+ (map (fn [part]
+ (if-let [error (when (= :error (:type part))
+ (provider-error provider nil (:error part)))]
+ (assoc part :error error)
+ part))))
(defn- missing-required-permission
"Returns the metabot permission keyword that the current user is missing
@@ -515,6 +539,7 @@
(core/lite-aisdk-xf)
(core/stamp-tool-titles-xf tools)
(report-aisdk-errors-xf tracking-opts)
+ (provider-error-xf provider)
(report-token-usage-xf tracking-opts)
(report-tool-usage-xf tracking-opts tools))
(stream-fn streaming-opts)))]
diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.clj
index ea2d4101d51..2905e12ea14 100644
--- a/src/metabase/metabot/self/claude.clj
+++ b/src/metabase/metabot/self/claude.clj
@@ -193,7 +193,8 @@
(= t "message_stop") identity
;; catch errors if any
(= t "error") (rf {:type :error
- :errorText (:message error)}))))))))
+ :errorText (:message error)
+ :error error}))))))))
;;; AISDK parts → Claude messages
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 945d465452b..d4753d0e0f2 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -421,7 +421,8 @@
;; Provider adapters emit AI SDK v5 error chunks (`{:type :error :errorText "..."}`). The rest of the
;; pipeline — `format-error-line` and `metabot.persistence`'s turn finalize — keys off an internal
;; `{:error {:message ...}}` part instead, so translate here. Internally-generated error parts (usage
- ;; limits, agent-loop exceptions) already carry an `:error` map; pass those through unchanged.
+ ;; limits, agent-loop exceptions) and Claude's error chunks, which keep the provider's own error
+ ;; object, already carry an `:error` map; pass those through unchanged.
:error (if (:error chunk)
chunk
{:type :error :error {:message (:errorText chunk)}})
diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.clj
index 2fe539988a1..ad4c60b889f 100644
--- a/src/metabase/slackbot/streaming.clj
+++ b/src/metabase/slackbot/streaming.clj
@@ -246,7 +246,7 @@
"You do not have permission to use the AI assistant."
(#{"metabase_ai_managed_locked" "ai_usage_limit_reached"
- "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"} code)
+ "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth" "ai_provider_unavailable"} code)
(:message error)
(provider-config-error-codes code)
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index f0ffd1a2174..520953d46cd 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -1003,36 +1003,49 @@
"Should have called LLM twice (1 failure + 1 success")))))))
(deftest provider-failure-error-part-test
- (let [credit-error (ex-info "Anthropic API error (HTTP 400)"
- {:status 400
- :body {:type "error"
- :error {:type "invalid_request_error"
- :message "Your credit balance is too low to access the Anthropic API."}}
- :api-error true
- :provider "anthropic"
- :error-code :provider-api-error})
- error-parts (fn [model-ref]
+ (let [api-error (fn [status error]
+ (ex-info (str "Anthropic API error (HTTP " status ")")
+ {:status status
+ :body {:type "error" :error error}
+ :api-error true
+ :provider "anthropic"
+ :error-code :provider-api-error}))
+ credit-error (api-error 400 {:type "invalid_request_error"
+ :message "Your credit balance is too low to access the Anthropic API."})
+ overloaded {:type "overloaded_error" :message "Overloaded"}
+ error-parts (fn [model-ref claude-fn]
(mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
llm-metabot-provider model-ref]
- (mt/with-dynamic-fn-redefs [claude/claude (fn [_] (throw credit-error))]
- (mt/with-log-level [metabase.metabot.agent.core :fatal]
+ (mt/with-dynamic-fn-redefs [claude/claude claude-fn
+ self/retry-delay-ms (constantly 0)]
+ (mt/with-log-level [metabase.metabot :fatal]
(filterv #(= :error (:type %))
(agent/run-agent-loop {:messages [{:role :user :content "Hi"}]
:state {}
:profile-id :embedding_next
- :context {}}))))))]
+ :context {}}))))))
+ throw-credit (fn [_] (throw credit-error))]
(testing "on the customer's own key, the error part says what to fix instead of passing the provider's text on"
(is (=? [{:error {:error-code "ai_provider_billing" :message #"Anthropic rejected the request .*"}}]
- (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6"))))
+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6" throw-credit))))
(is (=? [{:error {:error-code "ai_provider_billing" :message #"The AI provider rejected the request .*"}}]
(mt/with-current-user (mt/user->id :rasta)
- (error-parts "anthropic/claude-sonnet-4-6")))))
+ (error-parts "anthropic/claude-sonnet-4-6" throw-credit)))))
(testing "on the managed provider, the error part keeps today's generic shape"
(is (= [{:type :error
:error {:message (ex-message credit-error)
:type (str (type credit-error))
:data (ex-data credit-error)}}]
- (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6")))))))
+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6" throw-credit)))))
+ (testing "an outage says to try again later on either connection, whether the request failed or the stream did"
+ (doseq [claude-fn [(fn [_] (throw (api-error 529 overloaded)))
+ (fn [_] (eduction (claude/claude->aisdk-chunks-xf)
+ [{:type "message_start" :message {:id "msg_1"}}
+ {:type "error" :error overloaded}]))]]
+ (is (=? [{:error {:error-code "ai_provider_unavailable" :message #"Anthropic is having problems .*"}}]
+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6" claude-fn))))
+ (is (=? [{:error {:error-code "ai_provider_unavailable" :message #"The AI provider is having problems .*"}}]
+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6" claude-fn))))))))
;;; ===================== Prometheus Metrics Tests =====================
diff --git a/test/metabase/metabot/self/bedrock_test.clj b/test/metabase/metabot/self/bedrock_test.clj
index 858b240dc4a..f93759eb956 100644
--- a/test/metabase/metabot/self/bedrock_test.clj
+++ b/test/metabase/metabot/self/bedrock_test.clj
@@ -698,7 +698,9 @@
(chunk-message {:type "message_stop"
:amazon-bedrock-invocationMetrics {:inputTokenCount 10 :outputTokenCount 2}})])))))
(testing "an exception partway through becomes Claude's error chunk"
- (is (= [{:type :error :errorText "Model stream error"}]
+ (is (= [{:type :error
+ :errorText "Model stream error"
+ :error {:type "modelStreamErrorException" :message "Model stream error"}}]
(filter (comp #{:error} :type)
(runtime-chunks-for! [message-start stream-error]))))))
diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.clj
index d02a9ff6cc7..2dba56661c4 100644
--- a/test/metabase/metabot/self/claude_test.clj
+++ b/test/metabase/metabot/self/claude_test.clj
@@ -88,7 +88,7 @@
{:type "error" :error {:type "overloaded_error" :message "Overloaded"}}]
parts (into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)
err (m/find-first #(= :error (:type %)) parts)]
- (is (=? {:message "Overloaded"} (:error err)))
+ (is (=? {:type "overloaded_error" :message "Overloaded"} (:error err)))
(is (= (self.core/format-sse-event {:type "error" :errorText "Overloaded"})
(self.core/format-error-line err))))))
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index a0116dcb0fa..74a9feedd20 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -2601,46 +2601,50 @@
:error {:type "invalid_request_error"
:message "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits."}})
-(deftest byok-provider-error-test
+(deftest provider-error-test
(mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
llm-metabot-provider "anthropic/claude-sonnet-4-6"]
- (testing "classifies the failures that the customer can fix on their side"
+ (testing "classifies the failures that the customer can fix on their side, and the provider's outages"
(are [code provider status body]
- (= code (:error-code (mt/as-admin (self/byok-provider-error (provider-api-error! provider status body)))))
- "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body
- "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"
- :message "You have reached your specified API usage limits."}}
- "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}
- "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"
- :message "You have reached your API usage limits."
- :details {:error_code "enforced_spend_limit_reached"}}}
- "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"
- :code "insufficient_quota"
- :message "You exceeded your current quota."}}
- "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}
- "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}
- "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}
- "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}
- "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"
- :message "Your API key does not have permission to use the specified resource."}}
- nil "openai" 403 {:error {:type "request_forbidden"
- :code "unsupported_country_region_territory"
- :message "Country, region, or territory not supported"}}
- nil "anthropic" 400 {:error {:type "invalid_request_error"
- :message "max_tokens: Input should be greater than 0"}}
- nil "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}))
+ (= code (:error-code (mt/as-admin (self/provider-error (provider-api-error! provider status body)))))
+ "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body
+ "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"
+ :message "You have reached your specified API usage limits."}}
+ "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}
+ "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"
+ :message "You have reached your API usage limits."
+ :details {:error_code "enforced_spend_limit_reached"}}}
+ "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"
+ :code "insufficient_quota"
+ :message "You exceeded your current quota."}}
+ "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}
+ "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}
+ "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}
+ "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}
+ "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"
+ :message "Your API key does not have permission to use the specified resource."}}
+ "ai_provider_unavailable" "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}
+ "ai_provider_unavailable" "anthropic" 500 {:error {:type "api_error" :message "Internal server error"}}
+ "ai_provider_unavailable" "openai" 503 {:error {:message "The server is overloaded or not ready yet."}}
+ nil "openai" 403 {:error {:type "request_forbidden"
+ :code "unsupported_country_region_territory"
+ :message "Country, region, or territory not supported"}}
+ nil "anthropic" 400 {:error {:type "invalid_request_error"
+ :message "max_tokens: Input should be greater than 0"}}))
(testing "admins are told which provider failed, everyone else is not"
- (doseq [status [402 429 401]]
+ (doseq [status [402 429 401 529]]
(let [e (provider-api-error! "anthropic" status {})]
- (is (str/includes? (:message (mt/as-admin (self/byok-provider-error e)))
+ (is (str/includes? (:message (mt/as-admin (self/provider-error e)))
"Anthropic"))
(is (not (str/includes? (:message (mt/with-current-user (mt/user->id :rasta)
- (self/byok-provider-error e)))
+ (self/provider-error e)))
"Anthropic"))))))
- (testing "the managed provider keeps the generic error, since its failures are Metabase's to fix"
+ (testing "on the managed provider only an outage is explained, and without naming the provider"
(mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]
(is (nil? (mt/as-admin
- (self/byok-provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))))))
+ (self/provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))
+ (is (=? {:error-code "ai_provider_unavailable" :message #"The AI provider is having problems.*"}
+ (mt/as-admin (self/provider-error (provider-api-error! "anthropic" 529 {})))))))))
(deftest known-models-normalization-test
(testing "adapters that key model id to a map are passed through"
diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.clj
index 42cd3560e00..a3ed24f53b0 100644
--- a/test/metabase/slackbot/streaming_test.clj
+++ b/test/metabase/slackbot/streaming_test.clj
@@ -391,8 +391,8 @@
:type "clojure.lang.ExceptionInfo"
:data {:status-code 400 :api-error true :error-code :llm-not-configured}}})]
(is (str/includes? text "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."))))
- (testing "a provider failure the customer can fix keeps the message the agent loop wrote for it"
- (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"]]
+ (testing "a provider failure the agent loop explained keeps the message it wrote for it"
+ (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth" "ai_provider_unavailable"]]
(let [text (dm-error-part-appended-text!
{:type :error :error {:message "Ask your administrator to check the AI provider." :error-code code}})]
(is (str/includes? text "Ask your administrator to check the AI provider.") code))))