Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improving LLM error handling (#78003)

Nathan Voxlandd1f282210c37
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index ac25af491cc..5086d949001 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -113,16 +113,19 @@
(defn- parse-retry-after-header
"Extract retry-after seconds from response headers in ex-data, if present and ≤ 60s.
- Returns nil if not present or not a reasonable value."
+ Returns nil if not present or not a reasonable value. Handle repeated headers defensively so a
+ malformed header never turns a retryable error into an uncaught cast/parse error."
[^Exception e]
(when-let [headers (:headers (ex-data e))]
- (when-let [retry-after (or (get headers "retry-after")
- (get headers "Retry-After"))]
- (try
- (let [seconds (Long/parseLong retry-after)]
- (when (<= 0 seconds 60)
- (* seconds 1000)))
- (catch NumberFormatException _ nil)))))
+ (when-let [raw (or (get headers "retry-after")
+ (get headers "Retry-After"))]
+ (let [retry-after (if (sequential? raw) (first raw) raw)]
+ (when (string? retry-after)
+ (try
+ (let [seconds (Long/parseLong retry-after)]
+ (when (<= 0 seconds 60)
+ (* seconds 1000)))
+ (catch NumberFormatException _ nil)))))))
(defn- retry-delay-ms
"Calculate retry delay in milliseconds using exponential backoff with jitter.
@@ -236,34 +239,41 @@
(defn- with-retries
"Execute `(thunk)` with retry logic for transient LLM errors.
Retries up to `max-llm-retries` attempts with exponential backoff.
- Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels."
- [tracking-opts thunk]
- (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]
- (loop [attempt 1]
- (analytics/inc! :metabase-metabot/llm-requests labels)
- (let [timer (u/start-timer)
- result (try
- {:ok (thunk)}
- (catch Exception e
- (if (and (< attempt max-llm-retries)
- (retryable-error? e))
- (let [delay (retry-delay-ms attempt e)]
- (log/warn e "LLM call failed with retryable error, retrying"
- {:attempt attempt
- :max max-llm-retries
- :delay delay
- :status (:status (ex-data e))})
- (analytics/inc! :metabase-metabot/llm-retries labels)
- {:retry delay})
- (do (analytics/inc! :metabase-metabot/llm-errors
- (assoc labels :error-type (.getSimpleName (class e))))
- (throw e))))
- (finally
- (analytics/observe! :metabase-metabot/llm-duration-ms labels (u/since-ms timer))))]
- (if-let [delay (:retry result)]
- (do (Thread/sleep ^long delay)
- (recur (inc attempt)))
- (:ok result))))))
+ Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels.
+
+ `retry?` is an optional predicate on the caught exception, ANDed with
+ [[retryable-error?]]; returning false surfaces the error without retrying. The
+ streaming path passes one to avoid replaying a partially-consumed response."
+ ([tracking-opts thunk]
+ (with-retries tracking-opts thunk (constantly true)))
+ ([tracking-opts thunk retry?]
+ (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]
+ (loop [attempt 1]
+ (analytics/inc! :metabase-metabot/llm-requests labels)
+ (let [timer (u/start-timer)
+ result (try
+ {:ok (thunk)}
+ (catch Exception e
+ (if (and (< attempt max-llm-retries)
+ (retry? e)
+ (retryable-error? e))
+ (let [delay (retry-delay-ms attempt e)]
+ (log/warn e "LLM call failed with retryable error, retrying"
+ {:attempt attempt
+ :max max-llm-retries
+ :delay delay
+ :status (:status (ex-data e))})
+ (analytics/inc! :metabase-metabot/llm-retries labels)
+ {:retry delay})
+ (do (analytics/inc! :metabase-metabot/llm-errors
+ (assoc labels :error-type (.getSimpleName (class e))))
+ (throw e))))
+ (finally
+ (analytics/observe! :metabase-metabot/llm-duration-ms labels (u/since-ms timer))))]
+ (if-let [delay (:retry result)]
+ (do (Thread/sleep ^long delay)
+ (recur (inc attempt)))
+ (:ok result)))))))
(defn call-llm
"Call an LLM and stream processed parts.
@@ -316,9 +326,18 @@
:model model
:part-count (count parts)
:tool-count (count tools)}
- (with-retries
- tracking-opts
- #(reduce rf init (make-source)))))))))))
+ ;; `with-retries` re-runs its thunk on a retryable error, which re-opens the
+ ;; stream. That is safe only *before* any part has reached `rf`; once the consumer
+ ;; has seen output, replaying would duplicate it and re-execute tools. Gate retries
+ ;; on "nothing emitted yet" so a mid-stream failure surfaces instead of replaying.
+ (let [emitted? (volatile! false)
+ rf* (fn
+ ([acc] (rf acc))
+ ([acc x] (vreset! emitted? true) (rf acc x)))]
+ (with-retries
+ tracking-opts
+ #(reduce rf* init (make-source))
+ (fn [_e] (not @emitted?))))))))))))
(defn call-llm-structured
"Make an LLM call that returns structured JSON output.
@@ -364,7 +383,31 @@
result (some (fn [{:keys [type arguments]}]
(when (= type :tool-input)
arguments))
+ parts)
+ error (some (fn [{:keys [type error]}]
+ (when (= type :error)
+ error))
parts)]
- (or result
- (throw (ex-info "LLM returned no tool call in structured response"
- {:parts parts})))))))))
+ (cond
+ ;; The tool call's JSON failed to parse; `parse-tool-arguments` returned the
+ ;; `{:_raw_arguments ...}` sentinel. Reject it as invalid rather than handing a
+ ;; bogus map back to the caller as if it were a valid structured result.
+ (and (map? result) (contains? result :_raw_arguments))
+ (throw (ex-info "LLM returned malformed JSON in its structured tool call"
+ {:parts parts
+ :error-code "structured-output-invalid"
+ :raw-arguments (:_raw_arguments result)}))
+
+ result
+ result
+
+ ;; The provider failed mid-stream and emitted an `:error` part instead of throwing
+ ;; (e.g. an OpenAI `response.failed`). Surface its message and code so callers/logs
+ ;; see the real cause rather than a misleading "no tool call".
+ error
+ (throw (ex-info (or (:message error) "LLM stream returned an error")
+ {:parts parts :error error :error-code "llm-stream-error"}))
+
+ :else
+ (throw (ex-info "LLM returned no tool call in structured response"
+ {:parts parts})))))))))
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 9ca0ba52234..52b6040a22c 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -595,6 +595,19 @@
;; ignores Retry-After > 60s
500 1 (ex-info "err" {:status 429 :headers {"retry-after" "120"}}))))
+(deftest parse-retry-after-header-test
+ (let [parse #'self/parse-retry-after-header]
+ (testing "parses a plain string Retry-After into milliseconds"
+ (is (= 3000 (parse (ex-info "e" {:headers {"retry-after" "3"}}))))
+ (is (= 3000 (parse (ex-info "e" {:headers {"Retry-After" "3"}})))))
+ (testing "ignores values over 60s and non-numeric values"
+ (is (nil? (parse (ex-info "e" {:headers {"retry-after" "120"}}))))
+ (is (nil? (parse (ex-info "e" {:headers {"retry-after" "soon"}})))))
+ (testing "returns nil when the header is absent"
+ (is (nil? (parse (ex-info "e" {})))))
+ (testing "does not throw on a vector-valued header (duplicate Retry-After); uses the first value"
+ (is (= 3000 (parse (ex-info "e" {:headers {"retry-after" ["3" "5"]}})))))))
+
(deftest with-retries-test
(mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)]
(testing "succeeds on first attempt without retrying"
@@ -645,6 +658,87 @@
:ok)))))
(is (= 2 @calls))))))
+(defn- malformed-tool-input-response
+ "A reducible LLM stream whose forced tool call streams invalid JSON, so
+ `parse-tool-arguments` yields the `{:_raw_arguments ...}` sentinel."
+ []
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (reduce rf init [{:type :start :messageId "m1"}
+ {:type :tool-input-start :toolCallId "c1" :toolName "json"}
+ {:type :tool-input-delta :toolCallId "c1" :inputTextDelta "{not valid json"}
+ {:type :tool-input-available :toolCallId "c1" :toolName "json"}]))))
+
+(deftest call-llm-structured-rejects-malformed-json-test
+ (testing "malformed tool-call JSON is rejected as an error, not returned as a bogus result"
+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)
+ openrouter/openrouter (constantly (malformed-tool-input-response))]
+ (mt/with-log-level [metabase.metabot.self.core :fatal]
+ (let [e (try
+ (self/call-llm-structured "openrouter/test-model"
+ [{:role "user" :content "test"}]
+ {:type "object" :properties {:answer {:type "string"}}}
+ 0.3 1024 {:tag "metabot_agent"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (instance? clojure.lang.ExceptionInfo e)
+ "malformed JSON must throw, not return the {:_raw_arguments ...} sentinel as a result")
+ (is (= "structured-output-invalid" (:error-code (ex-data e)))))))))
+
+(deftest call-llm-structured-surfaces-provider-error-test
+ (testing "a provider mid-stream :error part surfaces its message, not a generic 'no tool call' error"
+ (mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)
+ openrouter/openrouter (constantly
+ (test-util/mock-llm-response
+ [{:type :error :errorText "content policy violation"}]))]
+ (mt/with-log-level [metabase.metabot.self :fatal]
+ (let [e (try
+ (self/call-llm-structured "openrouter/test-model"
+ [{:role "user" :content "test"}]
+ {:type "object" :properties {:answer {:type "string"}}}
+ 0.3 1024 {:tag "metabot_agent"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (instance? clojure.lang.ExceptionInfo e))
+ (is (re-find #"content policy violation" (ex-message e))
+ "the provider error message is surfaced, not hidden behind 'no tool call'")
+ (is (= "llm-stream-error" (:error-code (ex-data e)))))))))
+
+(deftest call-llm-does-not-replay-after-partial-emission-test
+ (testing "a retryable failure AFTER parts were emitted does not replay the stream (no duplicate output / re-run tools)"
+ (mt/with-dynamic-fn-redefs
+ [self/retry-delay-ms (constantly 0)
+ openrouter/openrouter (constantly
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ ;; emit one chunk, then fail mid-stream with an otherwise-retryable error
+ (let [_acc (rf init {:type :start :messageId "m1"})]
+ (throw (ex-info "mid-stream boom" {:status 500}))))))]
+ (mt/with-log-level [metabase.metabot.self :fatal]
+ (let [seen (atom [])
+ collect-rf (fn ([acc] acc) ([acc x] (swap! seen conj x) acc))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"mid-stream boom"
+ (reduce collect-rf nil (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"}))))
+ (is (= 1 (count (filter #(= :start (:type %)) @seen)))
+ "the :start part reaches the consumer exactly once, not once per retry attempt"))))))
+
+(deftest call-llm-still-retries-before-any-emission-test
+ (testing "a retryable failure BEFORE any part is emitted is still retried (clean replay)"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs
+ [self/retry-delay-ms (constantly 0)
+ openrouter/openrouter (fn [_opts]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (if (< (swap! calls inc) 3)
+ (throw (ex-info "rate limited" {:status 429}))
+ (reduce rf init (test-util/mock-llm-response [{:type :start :id "m1"}]))))))]
+ (mt/with-log-level [metabase.metabot.self :fatal]
+ (let [seen (atom [])]
+ (reduce (fn [acc x] (swap! seen conj x) acc) nil
+ (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"}))
+ (is (= 3 @calls) "retries until the pre-emission failures clear")
+ (is (= 1 (count (filter #(= :start (:type %)) @seen))))))))))
+
;;; ===================== Prometheus Metrics Tests =====================
(deftest call-llm-prometheus-test