Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix translation of aisdk error parts (#75913)

appleby3c173a9fcc2d
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 74fb125d100..1d47e322935 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -137,10 +137,17 @@
jitter)))
(defn- report-aisdk-errors-xf
- "Transducer that increments the llm-errors counter for :error parts in the aisdk stream."
+ "Transducer that logs and increments the llm-errors counter for :error parts in the aisdk stream."
[tracking-opts]
(map (fn [part]
(when (= (:type part) :error)
+ ;; A streamed `:error` part means the provider failed mid-response (e.g. an OpenAI
+ ;; `response.failed`) without throwing, so nothing else logs it. Surface it here so it
+ ;; shows up in the server logs alongside the metric and the persisted turn error.
+ (log/error "Metabot LLM stream returned an error"
+ {:model (:model tracking-opts "unknown")
+ :source (:tag tracking-opts "none")
+ :error (:error part)})
(analytics/inc! :metabase-metabot/llm-errors
{:model (:model tracking-opts "unknown")
:source (:tag tracking-opts "none")
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 471d45d71f9..86d00c36497 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -141,7 +141,13 @@
:start {:type :start
:id (:messageId chunk)}
:usage chunk
- :error chunk
+ ;; Provider adapters emit AI SDK v5 error chunks (`{:type :error :errorText "..."}`). The rest of the
+ ;; pipeline — `format-error-line` and `metabot.persistence`'s turn finalize — keys off an internal
+ ;; `{:error {:message ...}}` part instead, so translate here. Internally-generated error parts (usage
+ ;; limits, agent-loop exceptions) already carry an `:error` map; pass those through unchanged.
+ :error (if (:error chunk)
+ chunk
+ {:type :error :error {:message (:errorText chunk)}})
:text-start {:type :text
:id (:id chunk)
:text (->> (map :delta chunks)
diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.clj
index bcbab859940..0ddcc173293 100644
--- a/src/metabase/metabot/self/openai.clj
+++ b/src/metabase/metabot/self/openai.clj
@@ -114,14 +114,23 @@
:function_call {:type :tool-input-delta
:toolCallId (:toolCallId @payload)
:inputTextDelta delta}))
- (= (:type chunk)
- "response.completed") (rf {:type :usage
- :usage (let [u (:usage response)]
- {:promptTokens (:input_tokens u 0)
- :completionTokens (:output_tokens u 0)})
- ;; non-standard extension, not in AISDK5
- :id (:id response)
- :model @model-name})
+ ;; `response.completed` and `response.incomplete` are both terminal events carrying final usage.
+ ;; An incomplete response (e.g. truncated at max_output_tokens or stopped by a content filter)
+ ;; still has valid partial output, so we record its usage rather than treating it as an error.
+ (contains? #{"response.completed" "response.incomplete"} t)
+ (rf {:type :usage
+ :usage (let [u (:usage response)]
+ {:promptTokens (:input_tokens u 0)
+ :completionTokens (:output_tokens u 0)})
+ ;; non-standard extension, not in AISDK5
+ :id (:id response)
+ :model @model-name})
+ ;; `response.failed` is the Responses API's terminal failure event. Its error lives nested under
+ ;; `response.error`, not in a top-level `error` event, so surface it explicitly.
+ (= t "response.failed") (rf {:type :error
+ :errorText (or (get-in response [:error :message])
+ (get-in response [:error :code])
+ (tru "The model provider failed to complete the response"))})
(= t "error") (rf {:type :error
:errorText (or (:message error) (:message chunk))}))))))))
diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.clj
index c1de0e09a42..45ee665ba84 100644
--- a/test/metabase/metabot/persistence_test.clj
+++ b/test/metabase/metabot/persistence_test.clj
@@ -5,10 +5,12 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.persistence :as metabot-persistence]
[metabase.metabot.query-analyzer :as nqa]
+ [metabase.metabot.self.core :as self.core]
[metabase.metabot.used-tables :as used-tables]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.test.util :as tu]
+ [metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.log.capture :as log.capture]
[toucan2.core :as t2]))
@@ -601,6 +603,36 @@
(is (= "\"boom\"" (:error row)))
(is (= "boom" (:error chat-msg)))))))
+(deftest finalize-persists-streamed-error-text-part-to-error-column-test
+ ;; Regression guard: before the `aisdk-chunks->part` fix the error chunk passed through unchanged, so the
+ ;; part carried `:errorText` (not `:error`); api.clj's `(:error (u/seek ...))` extraction then returned nil
+ ;; and the error column silently stayed NULL — an errored turn looked like a success in the appdb.
+ (testing "a streamed AI SDK v5 :errorText part lands in the assistant row's error column as a {:message ...} map"
+ (t2/with-transaction [_conn nil {:rollback-only true}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [conversation-id (str (random-uuid))
+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!
+ conversation-id "internal"
+ {:role "user" :content "go"})
+ ;; Provider adapters emit AI SDK v5 chunks; `aisdk-xf` collects them into the parts
+ ;; vector that feeds finalize in `metabase.metabot.api`.
+ parts (into [] (self.core/aisdk-xf)
+ [{:type :start :messageId "m1"}
+ {:type :error
+ :errorText "The model provider failed to complete the response"}])
+ error-data (:error (u/seek #(= :error (:type %)) parts))]
+ (is (= {:message "The model provider failed to complete the response"} error-data)
+ "the error part carries an :error map (not a passed-through :errorText), so api.clj's extraction is non-nil")
+ (metabot-persistence/finalize-assistant-turn!
+ conversation-id assistant-msg-id parts
+ :error error-data)
+ (let [row (t2/select-one :model/MetabotMessage assistant-msg-id)]
+ (is (= {:message "The model provider failed to complete the response"}
+ (json/decode+kw (:error row)))
+ "the streamed errorText is persisted into the error column as a {:message ...} map")
+ (is (true? (:finished row))
+ "an errored turn still finalizes as finished")))))))
+
(deftest messages-chat-messages-skips-in-flight-placeholders-test
(testing "in-flight placeholders (assistant role, finished=nil, recent created_at)
are filtered out of the chat-message conversion, so a mid-stream read does not
diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.clj
index 5bc3fec3fe7..d23cc76939c 100644
--- a/test/metabase/metabot/self/claude_test.clj
+++ b/test/metabase/metabot/self/claude_test.clj
@@ -40,6 +40,15 @@
:cacheReadTokens nat-int?}}]
(into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))
+(deftest ^:parallel claude-error-event-uses-canonical-error-shape-test
+ (testing "an `error` SSE event becomes an :error part keyed by :error (read by the wire serializer + persistence)"
+ (let [raw [{:type "message_start" :message {:id "msg_1" :model "claude-haiku-4-5"}}
+ {:type "error" :error {:type "overloaded_error" :message "Overloaded"}}]
+ parts (into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)
+ err (m/find-first #(= :error (:type %)) parts)]
+ (is (=? {:message "Overloaded"} (:error err)))
+ (is (= "3:\"Overloaded\"" (self.core/format-error-line err))))))
+
(deftest ^:parallel claude-tool-input-conv-test
(let [raw-chunks (fixture "claude-tool-input"
{:input [{:role :user :content "What time is it in Kyiv?"}]
diff --git a/test/metabase/metabot/self/openai_test.clj b/test/metabase/metabot/self/openai_test.clj
index 111fe3a0338..438094156d5 100644
--- a/test/metabase/metabot/self/openai_test.clj
+++ b/test/metabase/metabot/self/openai_test.clj
@@ -116,6 +116,70 @@
:completionTokens pos-int?}}]
(into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) patched)))))))
+(deftest ^:parallel openai-response-failed-surfaces-error-test
+ (testing "a terminal response.failed event is surfaced as an :error chunk (not silently dropped)"
+ ;; The error detail lives nested under `response.error`, unlike a top-level `error` event. Without
+ ;; this branch a mid-stream failure ends silently after :start, with no error shown to the user.
+ (let [raw [{:type "response.created" :response {:id "resp_1" :model "gpt-5.5"}}
+ {:type "response.in_progress" :response {:id "resp_1"}}
+ {:type "response.failed"
+ :response {:id "resp_1"
+ :error {:code "server_error"
+ :message "The server had an error while processing your request. Sorry about that!"}}}]]
+ (is (=? [{:type :start}
+ {:type :error
+ :errorText "The server had an error while processing your request. Sorry about that!"}]
+ (into [] (openai/openai->aisdk-chunks-xf) raw))))))
+
+(deftest ^:parallel openai-response-failed-without-message-test
+ (testing "response.failed with no message falls back to the error code, then a generic message"
+ (let [code-only [{:type "response.created" :response {:id "resp_1"}}
+ {:type "response.failed" :response {:id "resp_1" :error {:code "server_error"}}}]
+ bare [{:type "response.created" :response {:id "resp_1"}}
+ {:type "response.failed" :response {:id "resp_1"}}]]
+ (is (=? [{:type :start} {:type :error :errorText "server_error"}]
+ (into [] (openai/openai->aisdk-chunks-xf) code-only)))
+ (is (=? [{:type :start} {:type :error :errorText "The model provider failed to complete the response"}]
+ (into [] (openai/openai->aisdk-chunks-xf) bare))))))
+
+(deftest ^:parallel openai-response-failed-reaches-wire-and-persistence-test
+ (testing "a response.failed error flows through to both the wire line and the persisted turn error"
+ ;; Regression guard: once collected into a part (via aisdk-chunks->part), the AI SDK v5 `:errorText`
+ ;; chunk must become an `{:error {:message ...}}` part, which is what self.core/format-error-line and
+ ;; metabot.api's finalize-turn `(:error part)` lookup both read.
+ (let [raw [{:type "response.created" :response {:id "resp_1" :model "gpt-5.5"}}
+ {:type "response.failed" :response {:id "resp_1" :error {:code "server_error" :message "boom"}}}]
+ parts (into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)
+ err (m/find-first #(= :error (:type %)) parts)]
+ (testing "persistence picks up a non-nil error payload"
+ (is (=? {:message "boom"} (:error err))))
+ (testing "wire serializer emits the message (not an empty string)"
+ (is (= "3:\"boom\"" (self.core/format-error-line err)))))))
+
+(deftest ^:parallel openai-response-incomplete-keeps-partial-text-and-usage-test
+ (testing "a terminal response.incomplete event keeps the partial text and still emits usage (not an error)"
+ ;; An incomplete response (e.g. truncated at max_output_tokens) has valid partial output, so we record
+ ;; its usage like a completed response rather than discarding it or surfacing an error.
+ (let [raw-chunks (fixture "openai-text"
+ {:input [{:role :user :content "Say hello briefly, in under 10 words."}]})
+ ;; turn the terminal response.completed into a response.incomplete carrying the same usage
+ patched (mapv (fn [chunk]
+ (if (= (:type chunk) "response.completed")
+ (-> chunk
+ (assoc :type "response.incomplete")
+ (assoc-in [:response :incomplete_details] {:reason "max_output_tokens"}))
+ chunk))
+ raw-chunks)
+ parts (into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) patched)]
+ (is (=? [{:type :start}
+ {:type :text :text string?}
+ {:type :usage
+ :usage {:promptTokens pos-int?
+ :completionTokens pos-int?}}]
+ parts))
+ (testing "no error chunk is produced for an incomplete (partial-but-valid) response"
+ (is (empty? (filter #(= :error (:type %)) parts)))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; Usage normalization tests
;;; ──────────────────────────────────────────────────────────────────