The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix translation of aisdk error parts (#75913)
appleby3c173a9fcc2d
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 74fb125d100..1d47e322935 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -137,10 +137,17 @@jitter)))(defn- report-aisdk-errors-xf- "Transducer that increments the llm-errors counter for :error parts in the aisdk stream."+ "Transducer that logs and increments the llm-errors counter for :error parts in the aisdk stream."[tracking-opts](map (fn [part](when (= (:type part) :error)+ ;; A streamed `:error` part means the provider failed mid-response (e.g. an OpenAI+ ;; `response.failed`) without throwing, so nothing else logs it. Surface it here so it+ ;; shows up in the server logs alongside the metric and the persisted turn error.+ (log/error "Metabot LLM stream returned an error"+ {:model (:model tracking-opts "unknown")+ :source (:tag tracking-opts "none")+ :error (:error part)})(analytics/inc! :metabase-metabot/llm-errors{:model (:model tracking-opts "unknown"):source (:tag tracking-opts "none")diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 471d45d71f9..86d00c36497 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -141,7 +141,13 @@:start {:type :start:id (:messageId chunk)}:usage chunk- :error chunk+ ;; Provider adapters emit AI SDK v5 error chunks (`{:type :error :errorText "..."}`). The rest of the+ ;; pipeline — `format-error-line` and `metabot.persistence`'s turn finalize — keys off an internal+ ;; `{:error {:message ...}}` part instead, so translate here. Internally-generated error parts (usage+ ;; limits, agent-loop exceptions) already carry an `:error` map; pass those through unchanged.+ :error (if (:error chunk)+ chunk+ {:type :error :error {:message (:errorText chunk)}}):text-start {:type :text:id (:id chunk):text (->> (map :delta chunks)diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.cljindex bcbab859940..0ddcc173293 100644--- a/src/metabase/metabot/self/openai.clj+++ b/src/metabase/metabot/self/openai.clj@@ -114,14 +114,23 @@:function_call {:type :tool-input-delta:toolCallId (:toolCallId @payload):inputTextDelta delta}))- (= (:type chunk)- "response.completed") (rf {:type :usage- :usage (let [u (:usage response)]- {:promptTokens (:input_tokens u 0)- :completionTokens (:output_tokens u 0)})- ;; non-standard extension, not in AISDK5- :id (:id response)- :model @model-name})+ ;; `response.completed` and `response.incomplete` are both terminal events carrying final usage.+ ;; An incomplete response (e.g. truncated at max_output_tokens or stopped by a content filter)+ ;; still has valid partial output, so we record its usage rather than treating it as an error.+ (contains? #{"response.completed" "response.incomplete"} t)+ (rf {:type :usage+ :usage (let [u (:usage response)]+ {:promptTokens (:input_tokens u 0)+ :completionTokens (:output_tokens u 0)})+ ;; non-standard extension, not in AISDK5+ :id (:id response)+ :model @model-name})+ ;; `response.failed` is the Responses API's terminal failure event. Its error lives nested under+ ;; `response.error`, not in a top-level `error` event, so surface it explicitly.+ (= t "response.failed") (rf {:type :error+ :errorText (or (get-in response [:error :message])+ (get-in response [:error :code])+ (tru "The model provider failed to complete the response"))})(= t "error") (rf {:type :error:errorText (or (:message error) (:message chunk))}))))))))diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.cljindex c1de0e09a42..45ee665ba84 100644--- a/test/metabase/metabot/persistence_test.clj+++ b/test/metabase/metabot/persistence_test.clj@@ -5,10 +5,12 @@[metabase.lib.metadata :as lib.metadata][metabase.metabot.persistence :as metabot-persistence][metabase.metabot.query-analyzer :as nqa]+ [metabase.metabot.self.core :as self.core][metabase.metabot.used-tables :as used-tables][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.test.util :as tu]+ [metabase.util :as u][metabase.util.json :as json][metabase.util.log.capture :as log.capture][toucan2.core :as t2]))@@ -601,6 +603,36 @@(is (= "\"boom\"" (:error row)))(is (= "boom" (:error chat-msg)))))))+(deftest finalize-persists-streamed-error-text-part-to-error-column-test+ ;; Regression guard: before the `aisdk-chunks->part` fix the error chunk passed through unchanged, so the+ ;; part carried `:errorText` (not `:error`); api.clj's `(:error (u/seek ...))` extraction then returned nil+ ;; and the error column silently stayed NULL — an errored turn looked like a success in the appdb.+ (testing "a streamed AI SDK v5 :errorText part lands in the assistant row's error column as a {:message ...} map"+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id "internal"+ {:role "user" :content "go"})+ ;; Provider adapters emit AI SDK v5 chunks; `aisdk-xf` collects them into the parts+ ;; vector that feeds finalize in `metabase.metabot.api`.+ parts (into [] (self.core/aisdk-xf)+ [{:type :start :messageId "m1"}+ {:type :error+ :errorText "The model provider failed to complete the response"}])+ error-data (:error (u/seek #(= :error (:type %)) parts))]+ (is (= {:message "The model provider failed to complete the response"} error-data)+ "the error part carries an :error map (not a passed-through :errorText), so api.clj's extraction is non-nil")+ (metabot-persistence/finalize-assistant-turn!+ conversation-id assistant-msg-id parts+ :error error-data)+ (let [row (t2/select-one :model/MetabotMessage assistant-msg-id)]+ (is (= {:message "The model provider failed to complete the response"}+ (json/decode+kw (:error row)))+ "the streamed errorText is persisted into the error column as a {:message ...} map")+ (is (true? (:finished row))+ "an errored turn still finalizes as finished")))))))+(deftest messages-chat-messages-skips-in-flight-placeholders-test(testing "in-flight placeholders (assistant role, finished=nil, recent created_at)are filtered out of the chat-message conversion, so a mid-stream read does notdiff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex 5bc3fec3fe7..d23cc76939c 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -40,6 +40,15 @@:cacheReadTokens nat-int?}}](into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw-chunks))))))+(deftest ^:parallel claude-error-event-uses-canonical-error-shape-test+ (testing "an `error` SSE event becomes an :error part keyed by :error (read by the wire serializer + persistence)"+ (let [raw [{:type "message_start" :message {:id "msg_1" :model "claude-haiku-4-5"}}+ {:type "error" :error {:type "overloaded_error" :message "Overloaded"}}]+ parts (into [] (comp (claude/claude->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)+ err (m/find-first #(= :error (:type %)) parts)]+ (is (=? {:message "Overloaded"} (:error err)))+ (is (= "3:\"Overloaded\"" (self.core/format-error-line err))))))+(deftest ^:parallel claude-tool-input-conv-test(let [raw-chunks (fixture "claude-tool-input"{:input [{:role :user :content "What time is it in Kyiv?"}]diff --git a/test/metabase/metabot/self/openai_test.clj b/test/metabase/metabot/self/openai_test.cljindex 111fe3a0338..438094156d5 100644--- a/test/metabase/metabot/self/openai_test.clj+++ b/test/metabase/metabot/self/openai_test.clj@@ -116,6 +116,70 @@:completionTokens pos-int?}}](into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) patched)))))))+(deftest ^:parallel openai-response-failed-surfaces-error-test+ (testing "a terminal response.failed event is surfaced as an :error chunk (not silently dropped)"+ ;; The error detail lives nested under `response.error`, unlike a top-level `error` event. Without+ ;; this branch a mid-stream failure ends silently after :start, with no error shown to the user.+ (let [raw [{:type "response.created" :response {:id "resp_1" :model "gpt-5.5"}}+ {:type "response.in_progress" :response {:id "resp_1"}}+ {:type "response.failed"+ :response {:id "resp_1"+ :error {:code "server_error"+ :message "The server had an error while processing your request. Sorry about that!"}}}]]+ (is (=? [{:type :start}+ {:type :error+ :errorText "The server had an error while processing your request. Sorry about that!"}]+ (into [] (openai/openai->aisdk-chunks-xf) raw))))))++(deftest ^:parallel openai-response-failed-without-message-test+ (testing "response.failed with no message falls back to the error code, then a generic message"+ (let [code-only [{:type "response.created" :response {:id "resp_1"}}+ {:type "response.failed" :response {:id "resp_1" :error {:code "server_error"}}}]+ bare [{:type "response.created" :response {:id "resp_1"}}+ {:type "response.failed" :response {:id "resp_1"}}]]+ (is (=? [{:type :start} {:type :error :errorText "server_error"}]+ (into [] (openai/openai->aisdk-chunks-xf) code-only)))+ (is (=? [{:type :start} {:type :error :errorText "The model provider failed to complete the response"}]+ (into [] (openai/openai->aisdk-chunks-xf) bare))))))++(deftest ^:parallel openai-response-failed-reaches-wire-and-persistence-test+ (testing "a response.failed error flows through to both the wire line and the persisted turn error"+ ;; Regression guard: once collected into a part (via aisdk-chunks->part), the AI SDK v5 `:errorText`+ ;; chunk must become an `{:error {:message ...}}` part, which is what self.core/format-error-line and+ ;; metabot.api's finalize-turn `(:error part)` lookup both read.+ (let [raw [{:type "response.created" :response {:id "resp_1" :model "gpt-5.5"}}+ {:type "response.failed" :response {:id "resp_1" :error {:code "server_error" :message "boom"}}}]+ parts (into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) raw)+ err (m/find-first #(= :error (:type %)) parts)]+ (testing "persistence picks up a non-nil error payload"+ (is (=? {:message "boom"} (:error err))))+ (testing "wire serializer emits the message (not an empty string)"+ (is (= "3:\"boom\"" (self.core/format-error-line err)))))))++(deftest ^:parallel openai-response-incomplete-keeps-partial-text-and-usage-test+ (testing "a terminal response.incomplete event keeps the partial text and still emits usage (not an error)"+ ;; An incomplete response (e.g. truncated at max_output_tokens) has valid partial output, so we record+ ;; its usage like a completed response rather than discarding it or surfacing an error.+ (let [raw-chunks (fixture "openai-text"+ {:input [{:role :user :content "Say hello briefly, in under 10 words."}]})+ ;; turn the terminal response.completed into a response.incomplete carrying the same usage+ patched (mapv (fn [chunk]+ (if (= (:type chunk) "response.completed")+ (-> chunk+ (assoc :type "response.incomplete")+ (assoc-in [:response :incomplete_details] {:reason "max_output_tokens"}))+ chunk))+ raw-chunks)+ parts (into [] (comp (openai/openai->aisdk-chunks-xf) (self.core/aisdk-xf)) patched)]+ (is (=? [{:type :start}+ {:type :text :text string?}+ {:type :usage+ :usage {:promptTokens pos-int?+ :completionTokens pos-int?}}]+ parts))+ (testing "no error chunk is produced for an incomplete (partial-but-valid) response"+ (is (empty? (filter #(= :error (:type %)) parts)))))))+;;; ──────────────────────────────────────────────────────────────────;;; Usage normalization tests;;; ──────────────────────────────────────────────────────────────────