The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Surface upstream body in LLM provider error logs and messages (#74359)
Chris Truter2368677accc6
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 4a98d1f60fb..e6f14ea3757 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -623,17 +623,23 @@result))(catch Exception e(analytics/inc! :metabase-metabot/agent-errors labels)- (if (:api-error (ex-data e))- (if (:status (ex-data e))- (log/errorf "Agent loop API error: %s status=%s provider=%s body=%s"- (ex-message e)- (:status (ex-data e))- (:provider (ex-data e))- (pr-str (:body (ex-data e))))- (log/errorf e "Agent loop API error: %s provider=%s"- (ex-message e)- (:provider (ex-data e))))- (log/error e "Agent loop error"))+ (let [{:keys [api-error status provider body]} (ex-data e)+ msg (ex-message e)]+ (cond+ (and api-error status)+ (log/errorf e "Agent loop API error: %s status=%s provider=%s body=%s"+ msg status provider (pr-str body))++ api-error+ (log/errorf e "Agent loop API error: %s provider=%s" msg provider)++ ;; ex-message can be nil/blank for exceptions thrown without a message+ ;; (e.g. (NullPointerException.)) — skip the colon when there's nothing to say.+ (str/blank? msg)+ (log/error e "Agent loop error")++ :else+ (log/errorf e "Agent loop error: %s" msg)))(rf init (error-part e)))(finally(analytics/observe! :metabase-metabot/agent-duration-ms labels (u/since-ms start-ms)))))))))))diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.cljindex 8ee0ca0e8c9..29799b6c321 100644--- a/src/metabase/metabot/self/claude.clj+++ b/src/metabase/metabot/self/claude.clj@@ -260,9 +260,10 @@{:type "text":text suffix}]))))-(defn- anthropic-errors [res]- (let [status (long (:status res 0))- error-msg (get-in res [:body :error :message])]+(defn- anthropic-error-msg+ "Canonical, status-specific Anthropic error message."+ [res]+ (let [status (long (:status res 0))](case status401 (tru "Anthropic API key expired or invalid")403 (tru "Anthropic API key has insufficient permissions")@@ -271,9 +272,7 @@429 (tru "Anthropic API has rate limited us")500 (tru "Anthropic API is not working but not saying why")529 (tru "Anthropic API is overloaded and is asking us to wait")- (if error-msg- (tru "Anthropic API error (HTTP {0}): {1}" status error-msg)- (tru "Anthropic API error (HTTP {0})" status)))))+ (tru "Anthropic API error (HTTP {0})" status))))(defn list-models"List available Anthropic models.@@ -295,7 +294,7 @@models (reverse (sort-by :created_at (:data body)))]{:models (map #(select-keys % [:id :display_name]) models)})(catch Exception e- (core/rethrow-api-error! "anthropic" anthropic-errors e)))))+ (core/rethrow-api-error! "anthropic" anthropic-error-msg e)))))(mu/defn claude-raw"Perform a streaming request to Claude API."@@ -347,7 +346,7 @@:url "/v1/messages":request req})))(catch Exception e- (core/rethrow-api-error! "anthropic" anthropic-errors e))))))+ (core/rethrow-api-error! "anthropic" anthropic-error-msg e))))))(defn claude"Call Claude API, return AISDK stream"diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 6c620ae4bc2..80ed4792985 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -11,7 +11,7 @@[metabase.util.log :as log][metabase.util.o11y :refer [with-span]])(:import- (java.io BufferedReader Closeable)+ (java.io BufferedReader Closeable InputStream)(java.util.concurrent Callable Executors ExecutorService)))(set! *warn-on-reflection* true)@@ -501,29 +501,175 @@(rf result chunk))))))+(def ^:private max-body-preview-chars+ "Cap on the body snippet spliced into provider error messages."+ 500)++(defn- extract-error-message+ "First non-blank string under `[:error :message]`, `:error`, `:detail`, or `:message`.+ Non-strings and whitespace-only strings fall through to the next key."+ [m]+ ;; Filter per-lookup so a structured value (e.g. {:error {:code 500}}) never gets+ ;; str-coerced into the user-facing exception message — bad shapes fall through to+ ;; the next key instead.+ (letfn [(s [v] (when (string? v) (not-empty (str/trim v))))]+ (or (s (get-in m [:error :message]))+ (s (:error m))+ (s (:detail m))+ (s (:message m)))))++(def ^:private body-slurp-chunk-chars+ "Read-chunk size for [[slurp-bounded]]. Keeps the transient buffer small so a tiny error+ envelope costs a tiny allocation; only a body that actually reaches the cap pays for it."+ 8192)++(defn- slurp-bounded+ "Read at most `limit` chars from `r` as UTF-8, returning nil on read error or empty input.+ Grows a `StringBuilder` in [[body-slurp-chunk-chars]] chunks so memory scales with the real+ body rather than pre-allocating the worst-case `limit`-sized buffer up front."+ [r limit]+ (try+ (with-open [rdr (io/reader r :encoding "UTF-8")]+ (let [buf (char-array body-slurp-chunk-chars)+ sb (StringBuilder.)]+ (loop []+ (let [remaining (- limit (.length sb))]+ (if (<= remaining 0)+ (str sb)+ (let [n (.read ^java.io.Reader rdr buf 0 (min remaining body-slurp-chunk-chars))]+ (if (neg? n)+ (when (pos? (.length sb)) (str sb))+ (do (.append sb buf 0 n)+ (recur)))))))))+ (catch Exception _ nil)))++(def ^:private max-body-slurp-chars+ "Cap on how many chars we read off an upstream InputStream body when decoding for error surfacing."+ ;; Large enough to cover any realistic JSON error envelope; small enough to bound the+ ;; pathological multi-MB case (e.g. a stuck stream from a misbehaving upstream).+ 1000000)++(def ^:private max-body-log-chars+ "Cap on the body `pr-str` spliced into warn/error log lines. Larger than the user-facing+ preview cap since operators want more context, but still bounded so a multi-MB body — or+ a near-cap slurped stream — can't flood the logs. The full body always survives in `ex-data`."+ 2000)++(defn- decode-bounded-body+ "Decode a clj-http response map's `:body` for error surfacing.+ Returns the response map with `:body` set to the decoded value, or — on parse failure —+ to the raw bounded string so [[body-preview]] still has something to surface."+ [res]+ ;; Bound-slurp InputStream bodies first so a giant upstream payload doesn't get pulled+ ;; fully into memory. On parse failure (e.g. the cap truncated us mid-envelope) we keep+ ;; the bounded string in :body rather than nil-ing it out.+ (let [bounded (cond-> res+ (instance? InputStream (:body res))+ (update :body #(or (slurp-bounded % max-body-slurp-chars) "")))]+ (try+ (json/decode-body bounded)+ (catch Exception _ bounded))))++(defn- truncate-to+ "Cap `s` at `limit` with a trailing ellipsis when it overflows."+ [s limit]+ (if (<= (count s) limit)+ s+ (str (subs s 0 limit) "…")))++(defn- truncate-to-preview-limit+ "Cap `s` at [[max-body-preview-chars]] with a trailing ellipsis when it overflows."+ [s]+ (truncate-to s max-body-preview-chars))++(defn- body-for-log+ "Bounded `pr-str` of a coerced body for warn/error log lines, capped at [[max-body-log-chars]]."+ [body]+ (truncate-to (pr-str body) max-body-log-chars))++(defn- body-preview+ "Short snippet of an upstream response body for the user-facing exception message.+ Non-empty maps/arrays without a recognised human-readable field fall back to `pr-str` and emit a warn.+ Nil/empty bodies return nil."+ [body]+ (let [extracted (cond+ (nil? body) nil+ (string? body) body+ (map? body) (extract-error-message body)+ (sequential? body) (let [head (first body)]+ (cond+ (map? head) (extract-error-message head)+ (string? head) head+ :else nil))+ :else nil)+ ;; Surface *some* context to the user even for unrecognised shapes — the warn+ ;; signals operators to add the new envelope shape to [[extract-error-message]].+ ;; Truncate once: the warn line shouldn't carry a multi-MB pr-str any more than+ ;; the user-facing exception message should.+ s (or extracted+ (when (and (or (map? body) (sequential? body)) (seq body))+ (let [capped (truncate-to-preview-limit (pr-str body))]+ (log/warnf "body-preview: unrecognised error body shape; pr-str=%s" capped)+ capped)))]+ (some-> s str/trim not-empty truncate-to-preview-limit)))++(def ^:private auth-error-statuses+ "Statuses whose upstream body may carry provider-side auth/account detail+ (raw API keys, org/account names, tenant IDs). The full body still hits the+ warn log; we just don't splice it into the message the caller sees."+ #{401 403})+(defn rethrow-api-error!"Rethrow a provider HTTP exception with a translated, user-facing message.-- `res->message` receives the decoded response map and must return the message- to surface to the client. If the exception already carries `:api-error true`- in its ex-data (e.g. a missing-API-key error from [[resolve-auth]]) it is- rethrown as-is so the original message is preserved."- [provider res->message e]+ `res->message` receives the decoded response map and returns the provider-specific message.+ A body preview is appended to the message except on 401/403 (see [[auth-error-statuses]]),+ where the body may carry sensitive auth/account detail; the full body is still logged.+ ex-data is an explicit allow-list of `:status`, `:reason-phrase`, `:headers`, `:body`, plus provider tags.+ Exceptions already tagged `:api-error true` are rethrown unchanged."+ [provider res->message ^Throwable e](let [data (ex-data e)](cond- (:api-error data) (throw e)- (:body data) (let [res (json/decode-body data)]- (throw (ex-info (res->message res)- (assoc res- :api-error true- :provider provider- :error-code :provider-api-error)- e)))- :else (throw (ex-info (tru "{0} API request failed: {1}" provider (ex-message e))- {:api-error true- :provider provider- :error-code :provider-request-failed}- e)))))+ (:api-error data)+ (throw e)++ (:body data)+ (let [res (decode-bounded-body data)+ base (res->message res)+ preview (when-not (contains? auth-error-statuses (:status res))+ (body-preview (:body res)))+ msg (cond-> base+ preview (str " — " preview))]+ ;; warnf (not warn) so the body renders into the message string, not as MDC.+ ;; body-for-log caps the pr-str so a near-cap slurped stream can't flood the logs;+ ;; the full body still survives in ex-data below.+ (log/warnf "Provider API request failed: provider=%s status=%s body=%s"+ provider (:status res) (body-for-log (:body res)))+ ;; Allow-list explicitly — clj-http responses carry :http-client (a Closeable),+ ;; :trace-redirects, :orig-content-encoding, etc., none of which should propagate downstream.+ ;; :headers is included so the retry path in metabase.metabot.self/parse-retry-after-header+ ;; can still honor Retry-After on 429/529 responses.+ (throw (ex-info msg+ (merge (select-keys res [:status :reason-phrase :headers :body])+ {:api-error true+ :provider provider+ :error-code :provider-api-error})+ e)))++ :else+ (let [exception-class (some-> e .getClass .getName)+ msg (ex-message e)]+ (log/warnf "Provider API request failed (no response): provider=%s class=%s message=%s"+ provider exception-class msg)+ ;; ex-message can be nil/blank for exceptions thrown without one (e.g. (RuntimeException.))+ ;; — skip the colon when the cause has nothing to say.+ (throw (ex-info (if (str/blank? msg)+ (tru "{0} API request failed" provider)+ (tru "{0} API request failed: {1}" provider msg))+ {:api-error true+ :provider provider+ :error-code :provider-request-failed+ :exception-class exception-class}+ e))))))(defn missing-api-key-ex"Create a standardized missing-API-key exception for provider adapters."diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.cljindex 77cbb197dd7..7667e08228d 100644--- a/src/metabase/metabot/self/openai.clj+++ b/src/metabase/metabot/self/openai.clj@@ -160,18 +160,17 @@:description doc:parameters (mjs/transform params {:additionalProperties false})}))-(defn- openai-errors [res]- (let [status (long (:status res 0))- error-msg (get-in res [:body :error :message])]+(defn- openai-error-msg+ "Canonical, status-specific OpenAI error message."+ [res]+ (let [status (long (:status res 0))](case status401 (tru "OpenAI API key expired or invalid")403 (tru "OpenAI API key has insufficient permissions")404 (tru "OpenAI API endpoint or model listing is unavailable")429 (tru "OpenAI API has rate limited us")500 (tru "OpenAI API is not working but not saying why")- (if error-msg- (tru "OpenAI API error (HTTP {0}): {1}" status error-msg)- (tru "OpenAI API error (HTTP {0})" status)))))+ (tru "OpenAI API error (HTTP {0})" status))))(defn list-models"List available OpenAI models.@@ -195,7 +194,7 @@:display_name (:id model)})(reverse (sort-by :created (get-in res [:body :data]))))})(catch Exception e- (core/rethrow-api-error! "openai" openai-errors e)))))+ (core/rethrow-api-error! "openai" openai-error-msg e)))))(mu/defn openai-raw"Perform a streaming request to OpenAI Responses API."@@ -239,7 +238,7 @@:url "/v1/responses":request req})))(catch Exception e- (core/rethrow-api-error! "openai" openai-errors e)))))+ (core/rethrow-api-error! "openai" openai-error-msg e)))))(defn openai"Call OpenAI API, return AISDK stream."diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.cljindex cc0d0f41531..767108c9fe4 100644--- a/src/metabase/metabot/self/openrouter.clj+++ b/src/metabase/metabot/self/openrouter.clj@@ -92,9 +92,10 @@:description doc:parameters (mjs/transform params {:additionalProperties false})}}))-(defn- openrouter-errors [res]- (let [status (long (:status res 0))- error-msg (get-in res [:body :error :message])]+(defn- openrouter-error-msg+ "Canonical, status-specific OpenRouter error message."+ [res]+ (let [status (long (:status res 0))](case status401 (tru "OpenRouter API key expired or invalid")402 (tru "OpenRouter has insufficient credits")@@ -104,9 +105,7 @@500 (tru "OpenRouter returned an internal server error")502 (tru "OpenRouter upstream provider returned an error")503 (tru "OpenRouter service is unavailable")- (if error-msg- (tru "OpenRouter API error (HTTP {0}): {1}" status error-msg)- (tru "OpenRouter API error (HTTP {0})" status)))))+ (tru "OpenRouter API error (HTTP {0})" status))))(defn list-models"List available OpenRouter models.@@ -132,7 +131,7 @@:display_name (or (:name model) (:id model))})(reverse (sort-by :created (get-in res [:body :data]))))})(catch Exception e- (core/rethrow-api-error! "openrouter" openrouter-errors e)))))+ (core/rethrow-api-error! "openrouter" openrouter-error-msg e)))));;; Streaming response → AISDK v5 chunks@@ -305,7 +304,7 @@:url "/v1/chat/completions":request req})))(catch Exception e- (core/rethrow-api-error! "openrouter" openrouter-errors e))))))+ (core/rethrow-api-error! "openrouter" openrouter-error-msg e))))))(defn openrouter"Call OpenRouter Chat Completions API, return AISDK stream."diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex e7170833110..d8284731861 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -12,6 +12,7 @@[metabase.metabot.test-util :as test-util][metabase.test :as mt][metabase.util.json :as json]+ [metabase.util.log.capture :as log.capture][ring.adapter.jetty :as jetty]))(set! *warn-on-reflection* true)@@ -53,10 +54,11 @@(testing "passes required tool choice to LLM providers"(let [captured (atom nil)](mt/with-premium-features #{:metabase-ai-managed}+ ;; `:api-error true` makes `rethrow-api-error!` rethrow as-is, so `::skip` survives on the outer ex-data.(mt/with-dynamic-fn-redefs [http/request (fn [opts](when (:body opts)(reset! captured (json/decode+kw (:body opts))))- (throw (ex-info "stop" {::skip true :status 401 :body "skip parsing"})))]+ (throw (ex-info "stop" {::skip true :api-error true})))](mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"llm-proxy-base-url "http://proxy.example"llm-openrouter-api-key "sk-or-v1-test-key"@@ -878,3 +880,272 @@"tag" "test-tag""session_id" "00000000-0000-0000-0000-000000000002"}}]token-events)))))))))++(deftest ^:parallel body-preview-test+ (let [body-preview #'self.core/body-preview]+ (testing "nil, blank, and non-string scalars → nil"+ (is (every? nil? (map body-preview [nil "" " " 500 :error true]))))+ (testing "plain strings pass through trimmed"+ (is (= "Internal Server Error" (body-preview " Internal Server Error "))))+ (testing "JSON envelopes prefer [:error :message] over :error/:detail/:message"+ (is (= "model decommissioned" (body-preview {:error {:message "model decommissioned" :type "x"}})))+ (is (= "invalid metric" (body-preview {:error "invalid metric"})))+ (is (= "missing prompt" (body-preview {:detail "missing prompt"})))+ (is (= "bad request" (body-preview {:message "bad request"}))))+ (testing "extract-error-message returns nil for non-string values at the recognised keys"+ (let [extract #'self.core/extract-error-message]+ (is (every? nil? (map extract [{:error {:code 42 :type "x"}}+ {:detail [{:loc ["body" "prompt"]}]}+ {:message {:code "missing"}}+ {:error {:message {:code 500}}}+ {:error 42}])))))+ (testing "a non-string, blank, or whitespace-only at one key falls through to a later key"+ (is (= "real error" (body-preview {:error {:message {:code 500}} :detail "real error"})))+ (is (= "real error" (body-preview {:error "" :detail "real error"})))+ (is (= "real error" (body-preview {:error " " :detail "real error"}))))+ (testing "empty maps and arrays return nil (nothing to preview, no warn)"+ (let [msgs (log.capture/with-log-messages-for-level [msgs [metabase.metabot.self.core :warn]]+ (is (every? nil? (map body-preview [{} []])))+ (msgs))]+ (is (empty? msgs))))+ (testing "non-empty maps/arrays without a recognised error field pr-str into the preview + warn"+ (let [bodies [{:request-id "abc" :trace ["frame1"]}+ [42 :kw]+ [{:request-id "abc"}]+ {:error {:code 42 :type "x"}}]+ msgs (log.capture/with-log-messages-for-level [msgs [metabase.metabot.self.core :warn]]+ (doseq [b bodies]+ (is (= (pr-str b) (body-preview b))+ (str "pr-str fallback for " (pr-str b))))+ (msgs))]+ (is (= (count bodies) (count msgs))+ "one warn line per pr-str fallback")+ (is (every? #(re-find #"unrecognised error body shape" (:message %)) msgs))))+ (testing "JSON arrays probe their first element"+ (is (= "rate limited" (body-preview [{:error {:message "rate limited"}} {:type "x"}])))+ (is (= "first message" (body-preview ["first message" "ignored"]))))+ (testing "long bodies are truncated to 500 chars with an ellipsis"+ (let [preview (body-preview (apply str (repeat 2000 \x)))]+ (is (str/ends-with? preview "…"))+ (is (= 501 (count preview)))))))++(defn- caught+ "Run `thunk` and return the thrown exception, or nil if it didn't throw."+ [thunk]+ (try (thunk) nil (catch Exception e e)))++(deftest rethrow-api-error!-passthrough-test+ (testing ":api-error exceptions are rethrown unchanged"+ (let [original (ex-info "boom" {:api-error true :error-code :proxy-not-configured})]+ (is (identical? original+ (caught #(self.core/rethrow-api-error! "anthropic" (constantly "X") original)))))))++(deftest rethrow-api-error!-string-body-test+ (testing "HTTP responses with a body get the upstream body appended and surfaced in ex-data"+ (let [upstream (ex-info "clj-http error"+ {:status 500+ :reason-phrase "Internal Server Error"+ :headers {"content-type" "application/json"}+ :body (json/encode {:error {:message "model decommissioned"}})+ :http-client (reify java.io.Closeable (close [_]))+ :trace-redirects ["http://elsewhere"]+ :orig-content-encoding "gzip"})+ ex (caught #(self.core/rethrow-api-error!+ "anthropic"+ (fn [res] (str "Anthropic API error (HTTP " (:status res) ")"))+ upstream))]+ (is (= "Anthropic API error (HTTP 500) — model decommissioned" (ex-message ex)))+ ;; pin exact ex-data keys — clj-http internals (:http-client, :trace-redirects, …) must not leak.+ (is (= #{:status :reason-phrase :headers :body :api-error :provider :error-code}+ (set (keys (ex-data ex)))))+ (is (=? {:api-error true :provider "anthropic" :error-code :provider-api-error+ :status 500 :body {:error {:message "model decommissioned"}}}+ (ex-data ex)))))++ (testing "non-JSON bodies still get a preview appended"+ (let [upstream (ex-info "clj-http error"+ {:status 502 :reason-phrase "Bad Gateway"+ :headers {"content-type" "text/plain"}+ :body "upstream gateway timeout"})+ ex (caught #(self.core/rethrow-api-error!+ "openrouter"+ (constantly "OpenRouter upstream provider returned an error")+ upstream))]+ (is (str/includes? (ex-message ex) "OpenRouter upstream provider returned an error"))+ (is (str/includes? (ex-message ex) "upstream gateway timeout"))+ (is (= #{:status :reason-phrase :headers :body :api-error :provider :error-code}+ (set (keys (ex-data ex)))))))++ (testing "structured maps without :error/:detail/:message pr-str into the user-facing message"+ (let [upstream (ex-info "clj-http error"+ {:status 500 :reason-phrase "Internal Server Error"+ :headers {"content-type" "application/json"}+ :body (json/encode {:request-id "abc" :trace ["frame1"]})})+ ex (caught #(self.core/rethrow-api-error!+ "anthropic"+ (constantly "Anthropic API is not working but not saying why")+ upstream))]+ (is (str/includes? (ex-message ex) "Anthropic API is not working but not saying why"))+ (is (str/includes? (ex-message ex) ":request-id")+ "the unrecognised envelope's pr-str is appended so operators see what we got")+ (is (= {:request-id "abc" :trace ["frame1"]} (:body (ex-data ex)))+ "the full body is still preserved in ex-data for debugging")+ (is (= #{:status :reason-phrase :headers :body :api-error :provider :error-code}+ (set (keys (ex-data ex))))))))++(deftest rethrow-api-error!-no-body-test+ (testing "non-HTTP errors (no :body) fall through to the request-failed branch"+ (let [ex (caught #(self.core/rethrow-api-error!+ "openai" (constantly "unused") (java.net.SocketTimeoutException. "Read timed out")))]+ (is (str/includes? (ex-message ex) "API request failed"))+ (is (str/includes? (ex-message ex) "Read timed out"))+ ;; pin exact ex-data keys for the no-body branch too.+ (is (= #{:api-error :provider :error-code :exception-class}+ (set (keys (ex-data ex)))))+ (is (=? {:api-error true :provider "openai" :error-code :provider-request-failed+ :exception-class "java.net.SocketTimeoutException"}+ (ex-data ex)))))++ (testing "no-body branch drops the trailing colon when ex-message is blank"+ (let [ex (caught #(self.core/rethrow-api-error! "openai" (constantly "unused") (RuntimeException.)))]+ (is (= "openai API request failed" (ex-message ex)))+ (is (= #{:api-error :provider :error-code :exception-class}+ (set (keys (ex-data ex))))))))++(deftest rethrow-api-error!-input-stream-test+ (testing "InputStream JSON bodies are decoded and structured-extracted"+ (let [json (json/encode {:error {:message "model decommissioned"}})+ upstream (ex-info "clj-http error"+ {:status 500 :reason-phrase "Internal Server Error"+ :headers {"content-type" "application/json"}+ :body (java.io.ByteArrayInputStream. (.getBytes ^String json))})+ ex (caught #(self.core/rethrow-api-error!+ "anthropic"+ (fn [res] (str "Anthropic API error (HTTP " (:status res) ")"))+ upstream))]+ (is (= "Anthropic API error (HTTP 500) — model decommissioned" (ex-message ex)))+ (is (=? {:error {:message "model decommissioned"}} (:body (ex-data ex))))))++ (testing "Large InputStream bodies are bounded — not fully slurped into memory"+ ;; ByteArrayInputStream.available() returns the unread byte count, so we can+ ;; measure how much rethrow-api-error! pulled off the stream without proxying.+ (let [body-bytes (.getBytes ^String (apply str (repeat 2000000 \x)))+ stream (java.io.ByteArrayInputStream. body-bytes)+ upstream (ex-info "clj-http error"+ {:status 502 :reason-phrase "Bad Gateway"+ :headers {"content-type" "text/plain"}+ :body stream})+ ex (caught #(self.core/rethrow-api-error!+ "openrouter"+ (constantly "OpenRouter upstream provider returned an error")+ upstream))+ consumed (- (alength body-bytes) (.available stream))]+ (is (str/includes? (ex-message ex) "OpenRouter upstream provider returned an error"))+ (is (str/ends-with? (ex-message ex) "…"))+ (is (< consumed (alength body-bytes))+ "should not consume the entire 2MB stream just to surface an error preview")))++ (testing "Truncated InputStream JSON bodies fall back to the raw bounded string"+ ;; A small slurp cap forces the JSON to be cut mid-envelope. We should fall back+ ;; to surfacing the raw bounded string rather than throwing on parse failure.+ (let [json (json/encode {:error {:message (apply str (repeat 10000 \x))}})+ upstream (ex-info "clj-http error"+ {:status 500 :reason-phrase "Internal Server Error"+ :headers {"content-type" "application/json"}+ :body (java.io.ByteArrayInputStream. (.getBytes ^String json))})+ ex (with-redefs [self.core/max-body-slurp-chars 100]+ (caught #(self.core/rethrow-api-error!+ "anthropic"+ (constantly "Anthropic upstream provider returned an error")+ upstream)))]+ (is (str/includes? (ex-message ex) "Anthropic upstream provider returned an error"))+ (is (str/includes? (ex-message ex) "{\"error\":{\"message\":\"xxx")+ "the truncated raw string is surfaced in the user-facing message when JSON parse fails")+ (is (string? (:body (ex-data ex)))+ "the bounded raw string is kept on ex-data when JSON parse fails")+ (is (<= (count (:body (ex-data ex))) 100)+ "the body in ex-data respects the slurp cap"))))++(deftest rethrow-api-error!-retry-after-test+ (testing "Retry-After header survives the ex-data allow-list and reaches retry-delay-ms"+ ;; Regression test: an earlier revision allow-listed only :status/:reason-phrase/:body,+ ;; which silently dropped :headers and made provider 429/529 retries fall back to+ ;; exponential backoff instead of honoring the upstream Retry-After.+ (let [upstream (ex-info "clj-http error"+ {:status 429 :reason-phrase "Too Many Requests"+ :headers {"retry-after" "3"}+ :body "rate limited"})+ ex (caught #(self.core/rethrow-api-error!+ "openrouter"+ (constantly "OpenRouter rate limit")+ upstream))]+ (is (= {"retry-after" "3"} (:headers (ex-data ex))))+ (is (<= 3000 (#'self/retry-delay-ms 1 ex) (+ 3000 750))+ "retry-delay-ms picks up the 3-second Retry-After through the rethrown exception"))))++(deftest rethrow-api-error!-warn-log-test+ (testing "the full upstream body is emitted at warn level alongside provider and status"+ (let [upstream (ex-info "clj-http error"+ {:status 502 :reason-phrase "Bad Gateway"+ :headers {"content-type" "text/plain"}+ :body "upstream gateway timeout"})+ [entry & more]+ (log.capture/with-log-messages-for-level [msgs [metabase.metabot.self.core :warn]]+ (caught #(self.core/rethrow-api-error!+ "openrouter"+ (constantly "OpenRouter upstream provider returned an error")+ upstream))+ (msgs))]+ (is (nil? more) "exactly one warn line at the failure boundary")+ (is (=? {:level :warn :namespace 'metabase.metabot.self.core}+ entry))+ (is (re-find #"provider=openrouter status=502 body=\"upstream gateway timeout\""+ (:message entry)))))+ (testing "an oversized body is capped in the warn log, but preserved in full on ex-data"+ (let [cap @#'self.core/max-body-log-chars+ big-body (apply str (repeat (+ cap 1000) \x))+ upstream (ex-info "clj-http error"+ {:status 502 :reason-phrase "Bad Gateway"+ :headers {"content-type" "text/plain"}+ :body big-body})+ [entry & more]+ (log.capture/with-log-messages-for-level [msgs [metabase.metabot.self.core :warn]]+ (let [ex (caught #(self.core/rethrow-api-error!+ "openrouter"+ (constantly "OpenRouter upstream provider returned an error")+ upstream))]+ (is (= big-body (:body (ex-data ex)))+ "the full, untruncated body still survives on ex-data"))+ (msgs))]+ (is (nil? more) "exactly one warn line at the failure boundary")+ (is (str/ends-with? (:message entry)+ (str "body=" (subs (pr-str big-body) 0 cap) "…"))+ "the warn line's body segment is capped at max-body-log-chars with a trailing ellipsis")+ (is (not (str/includes? (:message entry) big-body))+ "the full oversized body is not spliced into the warn line"))))++(deftest rethrow-api-error!-auth-status-body-not-leaked-test+ (testing "401/403 bodies are not appended to the user-facing message (may carry sensitive auth/account detail)"+ (doseq [status [401 403]]+ (let [secret "sk-leaked-key-abc123 for org=acme-corp tenant=42"+ upstream (ex-info "clj-http error"+ {:status status+ :reason-phrase "Unauthorized"+ :headers {"content-type" "application/json"}+ :body (json/encode {:error {:message secret}})})+ [entry & more]+ (log.capture/with-log-messages-for-level [msgs [metabase.metabot.self.core :warn]]+ (let [ex (caught #(self.core/rethrow-api-error!+ "anthropic"+ (fn [res] (str "Anthropic API error (HTTP " (:status res) ")"))+ upstream))]+ (is (= (str "Anthropic API error (HTTP " status ")") (ex-message ex))+ "no body preview spliced onto the user-facing message for auth statuses")+ (is (not (str/includes? (ex-message ex) secret))+ "secret-bearing body must not leak into the rethrown message")+ (is (= {:error {:message secret}} (:body (ex-data ex)))+ "the full decoded body is still preserved on ex-data for debugging"))+ (msgs))]+ (is (nil? more) "exactly one warn line at the failure boundary")+ (is (str/includes? (:message entry) secret)+ "the full body is still emitted at warn level for server-side debugging")))))