Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Explain BYOK provider failures in Metabot instead of a generic error (#83430)

Andrei Simionescu515ab1961fa6
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.ts
index b8a3bdace42..b07303cbcd9 100644
--- a/frontend/src/metabase/metabot/state/actions.ts
+++ b/frontend/src/metabase/metabot/state/actions.ts
@@ -719,7 +719,15 @@ export const sendAgentRequest = createAsyncThunk<
serverStarted,
error: streamedError,
display: isMatching(
- { type: "ai_usage_limit_reached", message: P.string },
+ {
+ type: P.union(
+ "ai_usage_limit_reached",
+ "ai_provider_billing",
+ "ai_provider_rate_limit",
+ "ai_provider_auth",
+ ),
+ message: P.string,
+ },
streamedError,
)
? // special case where we want to show the returned error from the backend
diff --git a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
index 43c010b72e6..1ae375743e3 100644
--- a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx
@@ -123,6 +123,35 @@ describe("metabot > errors", () => {
expect(await input()).toHaveTextContent("Who is your favorite?");
});
+ it.each([
+ "ai_provider_billing",
+ "ai_provider_rate_limit",
+ "ai_provider_auth",
+ ])("should show the backend message for %s errors", async (errorCode) => {
+ setup();
+ mockAgentEndpoint({
+ events: [
+ {
+ type: "error",
+ errorText:
+ "The AI provider rejected the request because of a billing issue. Please contact your administrator.",
+ },
+ {
+ type: "finish",
+ finishReason: "error",
+ messageMetadata: { errorCode },
+ },
+ ],
+ });
+
+ await enterChatMessage("Who is your favorite?");
+
+ await assertConversation([
+ ["user", "Who is your favorite?"],
+ ["agent", /The AI provider rejected the request because of a billing/],
+ ]);
+ });
+
it("should mask streamed errors with a generic message", async () => {
setup();
mockAgentEndpoint({ events: erroredResponse });
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index b00d9cf5118..8f5643184b0 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -530,7 +530,8 @@
{:type :text, :id (str (random-uuid)), :text message})
(defn- error-part [^Exception e]
- {:type :error, :error {:message (.getMessage e), :type (str (type e)), :data (ex-data e)}})
+ {:type :error, :error (or (self/byok-provider-error e)
+ {:message (.getMessage e), :type (str (type e)), :data (ex-data e)})})
(defn- accumulate-usage-xf
"Transducer that merges each `:usage` part into the cumulative usage atom
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 71991f1d61d..5edce0497f0 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -347,6 +347,54 @@
(recur (inc attempt)))
(:ok result)))))))
+(def ^:private provider-billing-error-codes
+ "Codes Anthropic and OpenAI put in an error body when the account is out of credit or over a spend limit."
+ #{"billing_error" "enforced_spend_limit_reached" "insufficient_quota" "credit_balance_exhausted"
+ "organization_spend_limit_exceeded" "project_spend_limit_exceeded" "organization_usage_limit_exceeded"})
+
+(defn- provider-failure
+ "Classify a provider API error's ex-data as `:billing`, `:rate-limit` or `:auth`, or nil for any other failure."
+ [{:keys [status body]}]
+ (let [{error-type :type error-code :code :keys [message details]} (:error body)]
+ (cond
+ (or (= status 402)
+ (some provider-billing-error-codes [error-type error-code (:error_code details)])
+ ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error
+ (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))
+ :billing
+
+ (= status 429)
+ :rate-limit
+
+ (or (= status 401) (= "permission_error" error-type))
+ :auth)))
+
+(defn byok-provider-error
+ "A user-facing `{:message :error-code}` for a provider failure that the customer can fix on their side, or nil.
+ Always nil on the managed provider, where these failures are Metabase's to fix. Only admins are told which
+ provider failed and where to fix it."
+ [e]
+ (let [{:keys [api-error provider] :as data} (ex-data e)]
+ (when-let [failure (and api-error
+ provider
+ (not (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider)))
+ (provider-failure data))]
+ (let [admin? api/*is-superuser?*
+ provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]
+ (case failure
+ :billing {:error-code "ai_provider_billing"
+ :message (if admin?
+ (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)
+ (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}
+ :rate-limit {:error-code "ai_provider_rate_limit"
+ :message (if admin?
+ (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)
+ (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}
+ :auth {:error-code "ai_provider_auth"
+ :message (if admin?
+ (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)
+ (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))})))))
+
(defn- missing-required-permission
"Returns the metabot permission keyword that the current user is missing
(the base `:permission/metabot` or `required-perm`), or nil when granted.
diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.clj
index f4615719982..bf16b150ff0 100644
--- a/src/metabase/slackbot/streaming.clj
+++ b/src/metabase/slackbot/streaming.clj
@@ -220,8 +220,9 @@
`:type :metabot/permission-denied` tag. `error` is a streamed `:error` part's
payload or a thrown exception's ex-data; errors the agent loop caught nest their
ex-data under `:data`. Only whitelisted markers get copy, and only the usage-limit
- codes pass their server-authored message through: everything else, raw provider
- errors and permission keywords included, must stay out of shared Slack channels."
+ and actionable provider-failure codes pass their server-authored message through:
+ everything else, raw provider errors and permission keywords included, must stay out
+ of shared Slack channels."
[error]
(let [code (some-> (or (:error-code error) (get-in error [:data :error-code])) name)]
(cond
@@ -230,7 +231,8 @@
(= :metabot/permission-denied (get-in error [:data :type])))
"You do not have permission to use the AI assistant."
- (#{"metabase_ai_managed_locked" "ai_usage_limit_reached"} code)
+ (#{"metabase_ai_managed_locked" "ai_usage_limit_reached"
+ "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"} code)
(:message error)
(provider-config-error-codes code)
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 210e68ea19b..5a12627b22c 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -16,6 +16,7 @@
[metabase.metabot.agent.profiles :as profiles]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.metabot.self :as self]
+ [metabase.metabot.self.claude :as claude]
[metabase.metabot.self.core :as self.core]
[metabase.metabot.self.openrouter :as openrouter]
[metabase.metabot.test-util :as mut]
@@ -899,6 +900,38 @@
(is (= 2 @call-count)
"Should have called LLM twice (1 failure + 1 success")))))))
+(deftest provider-failure-error-part-test
+ (let [credit-error (ex-info "Anthropic API error (HTTP 400)"
+ {:status 400
+ :body {:type "error"
+ :error {:type "invalid_request_error"
+ :message "Your credit balance is too low to access the Anthropic API."}}
+ :api-error true
+ :provider "anthropic"
+ :error-code :provider-api-error})
+ error-parts (fn [model-ref]
+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
+ llm-metabot-provider model-ref]
+ (mt/with-dynamic-fn-redefs [claude/claude (fn [_] (throw credit-error))]
+ (mt/with-log-level [metabase.metabot.agent.core :fatal]
+ (filterv #(= :error (:type %))
+ (agent/run-agent-loop {:messages [{:role :user :content "Hi"}]
+ :state {}
+ :profile-id :embedding_next
+ :context {}}))))))]
+ (testing "on the customer's own key, the error part says what to fix instead of passing the provider's text on"
+ (is (=? [{:error {:error-code "ai_provider_billing" :message #"Anthropic rejected the request .*"}}]
+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6"))))
+ (is (=? [{:error {:error-code "ai_provider_billing" :message #"The AI provider rejected the request .*"}}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (error-parts "anthropic/claude-sonnet-4-6")))))
+ (testing "on the managed provider, the error part keeps today's generic shape"
+ (is (= [{:type :error
+ :error {:message (ex-message credit-error)
+ :type (str (type credit-error))
+ :data (ex-data credit-error)}}]
+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6")))))))
+
;;; ===================== Prometheus Metrics Tests =====================
(deftest run-agent-loop-prometheus-test
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 0e7e736507d..ab79a191dbb 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -2332,6 +2332,63 @@
(is (not (str/includes? (:message entry) secret))
"the secret-bearing body never appears in the warn log")))))
+(defn- provider-api-error!
+ "What an adapter throws when `provider` answers with an HTTP error `status` and a JSON `body`."
+ [provider status body]
+ (mt/with-log-level [metabase.metabot.self.core :fatal]
+ (caught #(self.core/rethrow-api-error! provider
+ (constantly "API error")
+ (ex-info "clj-http error"
+ {:status status
+ :headers {"content-type" "application/json"}
+ :body (json/encode body)})))))
+
+(def ^:private anthropic-credit-balance-body
+ {:type "error"
+ :error {:type "invalid_request_error"
+ :message "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits."}})
+
+(deftest byok-provider-error-test
+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
+ llm-metabot-provider "anthropic/claude-sonnet-4-6"]
+ (testing "classifies the failures that the customer can fix on their side"
+ (are [code provider status body]
+ (= code (:error-code (mt/as-admin (self/byok-provider-error (provider-api-error! provider status body)))))
+ "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body
+ "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"
+ :message "You have reached your specified API usage limits."}}
+ "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}
+ "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"
+ :message "You have reached your API usage limits."
+ :details {:error_code "enforced_spend_limit_reached"}}}
+ "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"
+ :code "insufficient_quota"
+ :message "You exceeded your current quota."}}
+ "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}
+ "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}
+ "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}
+ "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}
+ "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"
+ :message "Your API key does not have permission to use the specified resource."}}
+ nil "openai" 403 {:error {:type "request_forbidden"
+ :code "unsupported_country_region_territory"
+ :message "Country, region, or territory not supported"}}
+ nil "anthropic" 400 {:error {:type "invalid_request_error"
+ :message "max_tokens: Input should be greater than 0"}}
+ nil "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}))
+ (testing "admins are told which provider failed, everyone else is not"
+ (doseq [status [402 429 401]]
+ (let [e (provider-api-error! "anthropic" status {})]
+ (is (str/includes? (:message (mt/as-admin (self/byok-provider-error e)))
+ "Anthropic"))
+ (is (not (str/includes? (:message (mt/with-current-user (mt/user->id :rasta)
+ (self/byok-provider-error e)))
+ "Anthropic"))))))
+ (testing "the managed provider keeps the generic error, since its failures are Metabase's to fix"
+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]
+ (is (nil? (mt/as-admin
+ (self/byok-provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))))))
+
(deftest known-models-normalization-test
(testing "adapters that key model id to a map are passed through"
(let [models (self/known-models "anthropic")]
diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.clj
index 2f85fb43b96..a5620aacb4a 100644
--- a/test/metabase/slackbot/streaming_test.clj
+++ b/test/metabase/slackbot/streaming_test.clj
@@ -390,6 +390,11 @@
:type "clojure.lang.ExceptionInfo"
:data {:status-code 400 :api-error true :error-code :llm-not-configured}}})]
(is (str/includes? text "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."))))
+ (testing "a provider failure the customer can fix keeps the message the agent loop wrote for it"
+ (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"]]
+ (let [text (dm-error-part-appended-text!
+ {:type :error :error {:message "Ask your administrator to check the AI provider." :error-code code}})]
+ (is (str/includes? text "Ask your administrator to check the AI provider.") code))))
(testing "an unrecognized error keeps the generic copy and leaks nothing from the provider"
(let [text (dm-error-part-appended-text!
{:type :error :error {:message "upstream rejected key sk-ant-oops"}})]