The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Explain BYOK provider failures in Metabot instead of a generic error (#83430)
Andrei Simionescu515ab1961fa6
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.tsindex b8a3bdace42..b07303cbcd9 100644--- a/frontend/src/metabase/metabot/state/actions.ts+++ b/frontend/src/metabase/metabot/state/actions.ts@@ -719,7 +719,15 @@ export const sendAgentRequest = createAsyncThunk<serverStarted,error: streamedError,display: isMatching(- { type: "ai_usage_limit_reached", message: P.string },+ {+ type: P.union(+ "ai_usage_limit_reached",+ "ai_provider_billing",+ "ai_provider_rate_limit",+ "ai_provider_auth",+ ),+ message: P.string,+ },streamedError,)? // special case where we want to show the returned error from the backenddiff --git a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsxindex 43c010b72e6..1ae375743e3 100644--- a/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/errors.unit.spec.tsx@@ -123,6 +123,35 @@ describe("metabot > errors", () => {expect(await input()).toHaveTextContent("Who is your favorite?");});+ it.each([+ "ai_provider_billing",+ "ai_provider_rate_limit",+ "ai_provider_auth",+ ])("should show the backend message for %s errors", async (errorCode) => {+ setup();+ mockAgentEndpoint({+ events: [+ {+ type: "error",+ errorText:+ "The AI provider rejected the request because of a billing issue. Please contact your administrator.",+ },+ {+ type: "finish",+ finishReason: "error",+ messageMetadata: { errorCode },+ },+ ],+ });++ await enterChatMessage("Who is your favorite?");++ await assertConversation([+ ["user", "Who is your favorite?"],+ ["agent", /The AI provider rejected the request because of a billing/],+ ]);+ });+it("should mask streamed errors with a generic message", async () => {setup();mockAgentEndpoint({ events: erroredResponse });diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex b00d9cf5118..8f5643184b0 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -530,7 +530,8 @@{:type :text, :id (str (random-uuid)), :text message})(defn- error-part [^Exception e]- {:type :error, :error {:message (.getMessage e), :type (str (type e)), :data (ex-data e)}})+ {:type :error, :error (or (self/byok-provider-error e)+ {:message (.getMessage e), :type (str (type e)), :data (ex-data e)})})(defn- accumulate-usage-xf"Transducer that merges each `:usage` part into the cumulative usage atomdiff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 71991f1d61d..5edce0497f0 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -347,6 +347,54 @@(recur (inc attempt)))(:ok result)))))))+(def ^:private provider-billing-error-codes+ "Codes Anthropic and OpenAI put in an error body when the account is out of credit or over a spend limit."+ #{"billing_error" "enforced_spend_limit_reached" "insufficient_quota" "credit_balance_exhausted"+ "organization_spend_limit_exceeded" "project_spend_limit_exceeded" "organization_usage_limit_exceeded"})++(defn- provider-failure+ "Classify a provider API error's ex-data as `:billing`, `:rate-limit` or `:auth`, or nil for any other failure."+ [{:keys [status body]}]+ (let [{error-type :type error-code :code :keys [message details]} (:error body)]+ (cond+ (or (= status 402)+ (some provider-billing-error-codes [error-type error-code (:error_code details)])+ ;; Anthropic reports a used-up credit balance or spend limit as a plain invalid_request_error+ (and (= status 400) (re-find #"credit balance|API usage limits" (str message))))+ :billing++ (= status 429)+ :rate-limit++ (or (= status 401) (= "permission_error" error-type))+ :auth)))++(defn byok-provider-error+ "A user-facing `{:message :error-code}` for a provider failure that the customer can fix on their side, or nil.+ Always nil on the managed provider, where these failures are Metabase's to fix. Only admins are told which+ provider failed and where to fix it."+ [e]+ (let [{:keys [api-error provider] :as data} (ex-data e)]+ (when-let [failure (and api-error+ provider+ (not (llm.provider/managed-model-ref? (metabot.settings/llm-metabot-provider)))+ (provider-failure data))]+ (let [admin? api/*is-superuser?*+ provider-name (or (some-> (llm.provider/provider-type provider) :label str) provider)]+ (case failure+ :billing {:error-code "ai_provider_billing"+ :message (if admin?+ (tru "{0} rejected the request because of a billing issue, such as running out of credits. Check the billing settings for your account." provider-name)+ (tru "The AI provider rejected the request because of a billing issue. Please contact your administrator."))}+ :rate-limit {:error-code "ai_provider_rate_limit"+ :message (if admin?+ (tru "{0} is rate limiting requests from Metabase. Try again in a moment, and if it keeps happening, check the rate limits for your account." provider-name)+ (tru "The AI provider is rate limiting requests right now. Please try again in a moment."))}+ :auth {:error-code "ai_provider_auth"+ :message (if admin?+ (tru "{0} rejected the API key or credentials that Metabase sent. Check them in the AI settings." provider-name)+ (tru "The AI provider rejected the credentials that Metabase sent. Please contact your administrator."))})))))+(defn- missing-required-permission"Returns the metabot permission keyword that the current user is missing(the base `:permission/metabot` or `required-perm`), or nil when granted.diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.cljindex f4615719982..bf16b150ff0 100644--- a/src/metabase/slackbot/streaming.clj+++ b/src/metabase/slackbot/streaming.clj@@ -220,8 +220,9 @@`:type :metabot/permission-denied` tag. `error` is a streamed `:error` part'spayload or a thrown exception's ex-data; errors the agent loop caught nest theirex-data under `:data`. Only whitelisted markers get copy, and only the usage-limit- codes pass their server-authored message through: everything else, raw provider- errors and permission keywords included, must stay out of shared Slack channels."+ and actionable provider-failure codes pass their server-authored message through:+ everything else, raw provider errors and permission keywords included, must stay out+ of shared Slack channels."[error](let [code (some-> (or (:error-code error) (get-in error [:data :error-code])) name)](cond@@ -230,7 +231,8 @@(= :metabot/permission-denied (get-in error [:data :type])))"You do not have permission to use the AI assistant."- (#{"metabase_ai_managed_locked" "ai_usage_limit_reached"} code)+ (#{"metabase_ai_managed_locked" "ai_usage_limit_reached"+ "ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"} code)(:message error)(provider-config-error-codes code)diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 210e68ea19b..5a12627b22c 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -16,6 +16,7 @@[metabase.metabot.agent.profiles :as profiles][metabase.metabot.persistence :as metabot.persistence][metabase.metabot.self :as self]+ [metabase.metabot.self.claude :as claude][metabase.metabot.self.core :as self.core][metabase.metabot.self.openrouter :as openrouter][metabase.metabot.test-util :as mut]@@ -899,6 +900,38 @@(is (= 2 @call-count)"Should have called LLM twice (1 failure + 1 success")))))))+(deftest provider-failure-error-part-test+ (let [credit-error (ex-info "Anthropic API error (HTTP 400)"+ {:status 400+ :body {:type "error"+ :error {:type "invalid_request_error"+ :message "Your credit balance is too low to access the Anthropic API."}}+ :api-error true+ :provider "anthropic"+ :error-code :provider-api-error})+ error-parts (fn [model-ref]+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections+ llm-metabot-provider model-ref]+ (mt/with-dynamic-fn-redefs [claude/claude (fn [_] (throw credit-error))]+ (mt/with-log-level [metabase.metabot.agent.core :fatal]+ (filterv #(= :error (:type %))+ (agent/run-agent-loop {:messages [{:role :user :content "Hi"}]+ :state {}+ :profile-id :embedding_next+ :context {}}))))))]+ (testing "on the customer's own key, the error part says what to fix instead of passing the provider's text on"+ (is (=? [{:error {:error-code "ai_provider_billing" :message #"Anthropic rejected the request .*"}}]+ (mt/as-admin (error-parts "anthropic/claude-sonnet-4-6"))))+ (is (=? [{:error {:error-code "ai_provider_billing" :message #"The AI provider rejected the request .*"}}]+ (mt/with-current-user (mt/user->id :rasta)+ (error-parts "anthropic/claude-sonnet-4-6")))))+ (testing "on the managed provider, the error part keeps today's generic shape"+ (is (= [{:type :error+ :error {:message (ex-message credit-error)+ :type (str (type credit-error))+ :data (ex-data credit-error)}}]+ (mt/as-admin (error-parts "metabase/anthropic/claude-sonnet-4-6")))))))+;;; ===================== Prometheus Metrics Tests =====================(deftest run-agent-loop-prometheus-testdiff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 0e7e736507d..ab79a191dbb 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -2332,6 +2332,63 @@(is (not (str/includes? (:message entry) secret))"the secret-bearing body never appears in the warn log")))))+(defn- provider-api-error!+ "What an adapter throws when `provider` answers with an HTTP error `status` and a JSON `body`."+ [provider status body]+ (mt/with-log-level [metabase.metabot.self.core :fatal]+ (caught #(self.core/rethrow-api-error! provider+ (constantly "API error")+ (ex-info "clj-http error"+ {:status status+ :headers {"content-type" "application/json"}+ :body (json/encode body)})))))++(def ^:private anthropic-credit-balance-body+ {:type "error"+ :error {:type "invalid_request_error"+ :message "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits."}})++(deftest byok-provider-error-test+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections+ llm-metabot-provider "anthropic/claude-sonnet-4-6"]+ (testing "classifies the failures that the customer can fix on their side"+ (are [code provider status body]+ (= code (:error-code (mt/as-admin (self/byok-provider-error (provider-api-error! provider status body)))))+ "ai_provider_billing" "anthropic" 400 anthropic-credit-balance-body+ "ai_provider_billing" "anthropic" 400 {:error {:type "invalid_request_error"+ :message "You have reached your specified API usage limits."}}+ "ai_provider_billing" "anthropic" 402 {:error {:type "billing_error" :message "Check your payment details."}}+ "ai_provider_billing" "anthropic" 429 {:error {:type "rate_limit_error"+ :message "You have reached your API usage limits."+ :details {:error_code "enforced_spend_limit_reached"}}}+ "ai_provider_billing" "openai" 429 {:error {:type "insufficient_quota"+ :code "insufficient_quota"+ :message "You exceeded your current quota."}}+ "ai_provider_billing" "openrouter" 402 {:error {:code 402 :message "Insufficient credits"}}+ "ai_provider_rate_limit" "anthropic" 429 {:error {:type "rate_limit_error" :message "Too many requests."}}+ "ai_provider_rate_limit" "openai" 429 {:error {:code "rate_limit_exceeded" :message "Rate limit reached."}}+ "ai_provider_auth" "anthropic" 401 {:error {:type "authentication_error" :message "invalid x-api-key"}}+ "ai_provider_auth" "anthropic" 403 {:error {:type "permission_error"+ :message "Your API key does not have permission to use the specified resource."}}+ nil "openai" 403 {:error {:type "request_forbidden"+ :code "unsupported_country_region_territory"+ :message "Country, region, or territory not supported"}}+ nil "anthropic" 400 {:error {:type "invalid_request_error"+ :message "max_tokens: Input should be greater than 0"}}+ nil "anthropic" 529 {:error {:type "overloaded_error" :message "Overloaded"}}))+ (testing "admins are told which provider failed, everyone else is not"+ (doseq [status [402 429 401]]+ (let [e (provider-api-error! "anthropic" status {})]+ (is (str/includes? (:message (mt/as-admin (self/byok-provider-error e)))+ "Anthropic"))+ (is (not (str/includes? (:message (mt/with-current-user (mt/user->id :rasta)+ (self/byok-provider-error e)))+ "Anthropic"))))))+ (testing "the managed provider keeps the generic error, since its failures are Metabase's to fix"+ (mt/with-temporary-setting-values [llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]+ (is (nil? (mt/as-admin+ (self/byok-provider-error (provider-api-error! "anthropic" 400 anthropic-credit-balance-body)))))))))+(deftest known-models-normalization-test(testing "adapters that key model id to a map are passed through"(let [models (self/known-models "anthropic")]diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex 2f85fb43b96..a5620aacb4a 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -390,6 +390,11 @@:type "clojure.lang.ExceptionInfo":data {:status-code 400 :api-error true :error-code :llm-not-configured}}})](is (str/includes? text "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."))))+ (testing "a provider failure the customer can fix keeps the message the agent loop wrote for it"+ (doseq [code ["ai_provider_billing" "ai_provider_rate_limit" "ai_provider_auth"]]+ (let [text (dm-error-part-appended-text!+ {:type :error :error {:message "Ask your administrator to check the AI provider." :error-code code}})]+ (is (str/includes? text "Ask your administrator to check the AI provider.") code))))(testing "an unrecognized error keeps the generic copy and leaks nothing from the provider"(let [text (dm-error-part-appended-text!{:type :error :error {:message "upstream rejected key sk-ant-oops"}})]