The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improving LLM error handling (#78003)
Nathan Voxlandd1f282210c37
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex ac25af491cc..5086d949001 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -113,16 +113,19 @@(defn- parse-retry-after-header"Extract retry-after seconds from response headers in ex-data, if present and ≤ 60s.- Returns nil if not present or not a reasonable value."+ Returns nil if not present or not a reasonable value. Handle repeated headers defensively so a+ malformed header never turns a retryable error into an uncaught cast/parse error."[^Exception e](when-let [headers (:headers (ex-data e))]- (when-let [retry-after (or (get headers "retry-after")- (get headers "Retry-After"))]- (try- (let [seconds (Long/parseLong retry-after)]- (when (<= 0 seconds 60)- (* seconds 1000)))- (catch NumberFormatException _ nil)))))+ (when-let [raw (or (get headers "retry-after")+ (get headers "Retry-After"))]+ (let [retry-after (if (sequential? raw) (first raw) raw)]+ (when (string? retry-after)+ (try+ (let [seconds (Long/parseLong retry-after)]+ (when (<= 0 seconds 60)+ (* seconds 1000)))+ (catch NumberFormatException _ nil)))))))(defn- retry-delay-ms"Calculate retry delay in milliseconds using exponential backoff with jitter.@@ -236,34 +239,41 @@(defn- with-retries"Execute `(thunk)` with retry logic for transient LLM errors.Retries up to `max-llm-retries` attempts with exponential backoff.- Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels."- [tracking-opts thunk]- (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]- (loop [attempt 1]- (analytics/inc! :metabase-metabot/llm-requests labels)- (let [timer (u/start-timer)- result (try- {:ok (thunk)}- (catch Exception e- (if (and (< attempt max-llm-retries)- (retryable-error? e))- (let [delay (retry-delay-ms attempt e)]- (log/warn e "LLM call failed with retryable error, retrying"- {:attempt attempt- :max max-llm-retries- :delay delay- :status (:status (ex-data e))})- (analytics/inc! :metabase-metabot/llm-retries labels)- {:retry delay})- (do (analytics/inc! :metabase-metabot/llm-errors- (assoc labels :error-type (.getSimpleName (class e))))- (throw e))))- (finally- (analytics/observe! :metabase-metabot/llm-duration-ms labels (u/since-ms timer))))]- (if-let [delay (:retry result)]- (do (Thread/sleep ^long delay)- (recur (inc attempt)))- (:ok result))))))+ Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels.++ `retry?` is an optional predicate on the caught exception, ANDed with+ [[retryable-error?]]; returning false surfaces the error without retrying. The+ streaming path passes one to avoid replaying a partially-consumed response."+ ([tracking-opts thunk]+ (with-retries tracking-opts thunk (constantly true)))+ ([tracking-opts thunk retry?]+ (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]+ (loop [attempt 1]+ (analytics/inc! :metabase-metabot/llm-requests labels)+ (let [timer (u/start-timer)+ result (try+ {:ok (thunk)}+ (catch Exception e+ (if (and (< attempt max-llm-retries)+ (retry? e)+ (retryable-error? e))+ (let [delay (retry-delay-ms attempt e)]+ (log/warn e "LLM call failed with retryable error, retrying"+ {:attempt attempt+ :max max-llm-retries+ :delay delay+ :status (:status (ex-data e))})+ (analytics/inc! :metabase-metabot/llm-retries labels)+ {:retry delay})+ (do (analytics/inc! :metabase-metabot/llm-errors+ (assoc labels :error-type (.getSimpleName (class e))))+ (throw e))))+ (finally+ (analytics/observe! :metabase-metabot/llm-duration-ms labels (u/since-ms timer))))]+ (if-let [delay (:retry result)]+ (do (Thread/sleep ^long delay)+ (recur (inc attempt)))+ (:ok result)))))))(defn call-llm"Call an LLM and stream processed parts.@@ -316,9 +326,18 @@:model model:part-count (count parts):tool-count (count tools)}- (with-retries- tracking-opts- #(reduce rf init (make-source)))))))))))+ ;; `with-retries` re-runs its thunk on a retryable error, which re-opens the+ ;; stream. That is safe only *before* any part has reached `rf`; once the consumer+ ;; has seen output, replaying would duplicate it and re-execute tools. Gate retries+ ;; on "nothing emitted yet" so a mid-stream failure surfaces instead of replaying.+ (let [emitted? (volatile! false)+ rf* (fn+ ([acc] (rf acc))+ ([acc x] (vreset! emitted? true) (rf acc x)))]+ (with-retries+ tracking-opts+ #(reduce rf* init (make-source))+ (fn [_e] (not @emitted?))))))))))))(defn call-llm-structured"Make an LLM call that returns structured JSON output.@@ -364,7 +383,31 @@result (some (fn [{:keys [type arguments]}](when (= type :tool-input)arguments))+ parts)+ error (some (fn [{:keys [type error]}]+ (when (= type :error)+ error))parts)]- (or result- (throw (ex-info "LLM returned no tool call in structured response"- {:parts parts})))))))))+ (cond+ ;; The tool call's JSON failed to parse; `parse-tool-arguments` returned the+ ;; `{:_raw_arguments ...}` sentinel. Reject it as invalid rather than handing a+ ;; bogus map back to the caller as if it were a valid structured result.+ (and (map? result) (contains? result :_raw_arguments))+ (throw (ex-info "LLM returned malformed JSON in its structured tool call"+ {:parts parts+ :error-code "structured-output-invalid"+ :raw-arguments (:_raw_arguments result)}))++ result+ result++ ;; The provider failed mid-stream and emitted an `:error` part instead of throwing+ ;; (e.g. an OpenAI `response.failed`). Surface its message and code so callers/logs+ ;; see the real cause rather than a misleading "no tool call".+ error+ (throw (ex-info (or (:message error) "LLM stream returned an error")+ {:parts parts :error error :error-code "llm-stream-error"}))++ :else+ (throw (ex-info "LLM returned no tool call in structured response"+ {:parts parts})))))))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 9ca0ba52234..52b6040a22c 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -595,6 +595,19 @@;; ignores Retry-After > 60s500 1 (ex-info "err" {:status 429 :headers {"retry-after" "120"}}))))+(deftest parse-retry-after-header-test+ (let [parse #'self/parse-retry-after-header]+ (testing "parses a plain string Retry-After into milliseconds"+ (is (= 3000 (parse (ex-info "e" {:headers {"retry-after" "3"}}))))+ (is (= 3000 (parse (ex-info "e" {:headers {"Retry-After" "3"}})))))+ (testing "ignores values over 60s and non-numeric values"+ (is (nil? (parse (ex-info "e" {:headers {"retry-after" "120"}}))))+ (is (nil? (parse (ex-info "e" {:headers {"retry-after" "soon"}})))))+ (testing "returns nil when the header is absent"+ (is (nil? (parse (ex-info "e" {})))))+ (testing "does not throw on a vector-valued header (duplicate Retry-After); uses the first value"+ (is (= 3000 (parse (ex-info "e" {:headers {"retry-after" ["3" "5"]}})))))))+(deftest with-retries-test(mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)](testing "succeeds on first attempt without retrying"@@ -645,6 +658,87 @@:ok)))))(is (= 2 @calls))))))+(defn- malformed-tool-input-response+ "A reducible LLM stream whose forced tool call streams invalid JSON, so+ `parse-tool-arguments` yields the `{:_raw_arguments ...}` sentinel."+ []+ (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (reduce rf init [{:type :start :messageId "m1"}+ {:type :tool-input-start :toolCallId "c1" :toolName "json"}+ {:type :tool-input-delta :toolCallId "c1" :inputTextDelta "{not valid json"}+ {:type :tool-input-available :toolCallId "c1" :toolName "json"}]))))++(deftest call-llm-structured-rejects-malformed-json-test+ (testing "malformed tool-call JSON is rejected as an error, not returned as a bogus result"+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)+ openrouter/openrouter (constantly (malformed-tool-input-response))]+ (mt/with-log-level [metabase.metabot.self.core :fatal]+ (let [e (try+ (self/call-llm-structured "openrouter/test-model"+ [{:role "user" :content "test"}]+ {:type "object" :properties {:answer {:type "string"}}}+ 0.3 1024 {:tag "metabot_agent"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (instance? clojure.lang.ExceptionInfo e)+ "malformed JSON must throw, not return the {:_raw_arguments ...} sentinel as a result")+ (is (= "structured-output-invalid" (:error-code (ex-data e)))))))))++(deftest call-llm-structured-surfaces-provider-error-test+ (testing "a provider mid-stream :error part surfaces its message, not a generic 'no tool call' error"+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)+ openrouter/openrouter (constantly+ (test-util/mock-llm-response+ [{:type :error :errorText "content policy violation"}]))]+ (mt/with-log-level [metabase.metabot.self :fatal]+ (let [e (try+ (self/call-llm-structured "openrouter/test-model"+ [{:role "user" :content "test"}]+ {:type "object" :properties {:answer {:type "string"}}}+ 0.3 1024 {:tag "metabot_agent"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (instance? clojure.lang.ExceptionInfo e))+ (is (re-find #"content policy violation" (ex-message e))+ "the provider error message is surfaced, not hidden behind 'no tool call'")+ (is (= "llm-stream-error" (:error-code (ex-data e)))))))))++(deftest call-llm-does-not-replay-after-partial-emission-test+ (testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"+ (mt/with-dynamic-fn-redefs+ [self/retry-delay-ms (constantly 0)+ openrouter/openrouter (constantly+ (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ ;; emit one chunk, then fail mid-stream with an otherwise-retryable error+ (let [_acc (rf init {:type :start :messageId "m1"})]+ (throw (ex-info "mid-stream boom" {:status 500}))))))]+ (mt/with-log-level [metabase.metabot.self :fatal]+ (let [seen (atom [])+ collect-rf (fn ([acc] acc) ([acc x] (swap! seen conj x) acc))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"mid-stream boom"+ (reduce collect-rf nil (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"}))))+ (is (= 1 (count (filter #(= :start (:type %)) @seen)))+ "the :start part reaches the consumer exactly once, not once per retry attempt"))))))++(deftest call-llm-still-retries-before-any-emission-test+ (testing "a retryable failure BEFORE any part is emitted is still retried (clean replay)"+ (let [calls (atom 0)]+ (mt/with-dynamic-fn-redefs+ [self/retry-delay-ms (constantly 0)+ openrouter/openrouter (fn [_opts]+ (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (if (< (swap! calls inc) 3)+ (throw (ex-info "rate limited" {:status 429}))+ (reduce rf init (test-util/mock-llm-response [{:type :start :id "m1"}]))))))]+ (mt/with-log-level [metabase.metabot.self :fatal]+ (let [seen (atom [])]+ (reduce (fn [acc x] (swap! seen conj x) acc) nil+ (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"}))+ (is (= 3 @calls) "retries until the pre-emission failures clear")+ (is (= 1 (count (filter #(= :start (:type %)) @seen))))))))))+;;; ===================== Prometheus Metrics Tests =====================(deftest call-llm-prometheus-test