The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-522: Seed the Slack agent loop with persisted conversation state (#80204)
Paolo Rascunà9cd6391ce925
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex cdc0473eed6..1606e27e493 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -353,7 +353,8 @@;;; ---------------------------------------- Conversation state -----------------------------------------(defn- replayable-assistant-row?+(defn replayable-assistant-row?+ "Is `row` an assistant message a later turn may replay?"[{:keys [error finished] :as row}](and (assistant-row? row)(nil? error)diff --git a/src/metabase/slackbot/persistence.clj b/src/metabase/slackbot/persistence.cljindex 1e1af931ca0..21b5b99104d 100644--- a/src/metabase/slackbot/persistence.clj+++ b/src/metabase/slackbot/persistence.clj@@ -18,7 +18,8 @@(into [] (mapcat #(metabot.persistence/tool-part->llm-messages % {:on-unresolved :skip})))))(defn message-history- "Tool call history for Slack messages. Returns {slack-msg-id -> [messages...]}."+ "Tool call history for Slack messages. Returns {slack-msg-id -> [messages...]}.+ Only [[metabase.metabot.persistence/replayable-assistant-row?]] rows contribute."[conversation-id slack-msg-ids](when (seq slack-msg-ids)(->> (t2/select :model/MetabotMessage@@ -26,6 +27,9 @@:role "assistant":deleted_at nil:slack_msg_id [:in slack-msg-ids])+ ;; A failed turn's state is dropped by [[metabase.metabot.persistence/conversation-state]], so+ ;; replaying its tool calls would announce queries the seeded state does not contain.+ (filter metabot.persistence/replayable-assistant-row?)(keep (fn [{:keys [slack_msg_id] :as msg}](when-let [parts (seq (extract-history-messages msg))][slack_msg_id parts])))@@ -42,6 +46,20 @@:deleted_at [:not= nil]:slack_msg_id [:in slack-msg-ids])))+(defn state-messages+ "The assistant rows of a Slack thread in reader order, for feeding+ [[metabase.metabot.persistence/conversation-state]].++ Carries only the columns that function filters and merges on, so a long thread's+ `data` blobs aren't loaded just to get at `state`. Not usable for history replay —+ see [[message-history]] for that."+ [conversation-id]+ (t2/select [:model/MetabotMessage :id :role :state :error :finished]+ :conversation_id conversation-id+ :role "assistant"+ :deleted_at nil+ {:order-by [[:created_at :asc] [:id :asc]]}))+(defn response-owner-user-id"Find the Metabase user ID who triggered the assistant response for this Slack channel/message.Returns nil when the message is not tracked."diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.cljindex 34deb376915..0f561e94ce9 100644--- a/src/metabase/slackbot/streaming.clj+++ b/src/metabase/slackbot/streaming.clj@@ -222,6 +222,15 @@:slack-msg-id req-slack-msg-id:user-id api/*current-user-id*:ai-proxy? ai-proxy?))+ ;; Slack replays message text from the thread itself, but the state earlier turns+ ;; produced only ever existed in the app DB — without it a follow-up cannot resolve+ ;; an id an earlier turn generated. Mirrors the in-app path in [[metabase.metabot.api]].+ ;;+ ;; Deliberately outside the lock's transaction: this turn's own rows are excluded+ ;; anyway (user row by role, placeholder by NULL `finished`), and a failure here+ ;; must not roll back the turn we just persisted (BOT-1279).+ state-messages (slackbot.persistence/state-messages conversation-id)+ baseline-state (metabot.persistence/conversation-state state-messages)data-idx (volatile! -1)request-message (metabot.envelope/user-message (or request-prompt prompt))capabilities (compute-capabilities)@@ -276,7 +285,7 @@(transduce dispatch-xf (constantly nil) nil(agent/run-agent-loop{:messages messages- :state {}+ :state baseline-state:profile-id :slackbot:conversation-id conversation-id:context context@@ -294,13 +303,19 @@;; UPDATE the placeholder with whatever parts we collected, even if the;; pipeline threw. Raw native parts (not the lossy AI-SDK-message;; round-trip) preserve tool-output :structured-output for analytics.- (metabot.persistence/finalize-assistant-turn!- assistant-msg-id- (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)- :profile-id "slackbot"- :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))- :turn-state (some-> @memory-atom memory/turn-state)- :error (some-> @thrown metabot.persistence/throwable->error-payload))))+ (let [combined-parts (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)]+ (metabot.persistence/finalize-assistant-turn!+ assistant-msg-id+ combined-parts+ :profile-id "slackbot"+ :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))+ :turn-state (some-> @memory-atom memory/turn-state)+ ;; A thrown error is more authoritative, but the agent loop catches most+ ;; failures internally and emits an `:error` part instead of throwing. Without+ ;; the fallback such a turn persists as a clean `finished` row, and+ ;; `conversation-state` then merges its partial state into every later turn.+ :error (or (some-> @thrown metabot.persistence/throwable->error-payload)+ (:error (u/seek #(= :error (:type %)) combined-parts))))))){:msg-id assistant-msg-id:external-id assistant-external-id}))diff --git a/test/metabase/slackbot/persistence_test.clj b/test/metabase/slackbot/persistence_test.cljindex c22e690de6e..2a234f99572 100644--- a/test/metabase/slackbot/persistence_test.clj+++ b/test/metabase/slackbot/persistence_test.clj@@ -1,6 +1,7 @@(ns metabase.slackbot.persistence-test(:require[clojure.test :refer :all]+ [metabase.metabot.persistence :as metabot.persistence][metabase.slackbot.persistence :as slackbot.persistence][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -79,6 +80,49 @@(is (= #{deleted-ts}(slackbot.persistence/deleted-message-ids conv-id #{deleted-ts}))))))))+(deftest message-history-excludes-non-replayable-rows-test+ (testing "only rows a later turn may replay contribute tool history"+ (let [conv-id (str (random-uuid))+ insert! (fn [slack-ts call-id & {:keys [error] :as opts}]+ (t2/insert! :model/MetabotMessage+ (cond-> {:conversation_id conv-id+ :slack_msg_id slack-ts+ :role "assistant"+ :profile_id "slackbot"+ :total_tokens 0+ :data [{:type "tool-search"+ :toolCallId call-id+ :state "output-available"+ :input {}+ :output {:output "result"}}]+ :data_version 2}+ (contains? opts :finished) (assoc :finished (:finished opts))+ error (assoc :error error))))+ clean "1712100000.000001"+ errored "1712100000.000002"+ in-flight "1712100000.000003"+ legacy "1712100000.000004"]+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})+ (insert! clean "call-clean" :finished true)+ (insert! errored "call-errored" :finished true :error "boom")+ (insert! in-flight "call-in-flight" :finished nil)+ ;; No :finished at all -- exercises the column default that keeps pre-migration threads readable.+ (insert! legacy "call-legacy")+ (let [history (slackbot.persistence/message-history+ conv-id #{clean errored in-flight legacy})]+ (testing "a clean finished row still contributes -- guards against over-filtering"+ (is (contains? history clean)))+ (testing "an errored row contributes nothing"+ (is (not (contains? history errored))))+ ;; Unreachable in production: insert-assistant-placeholder! writes neither `finished` nor+ ;; `slack_msg_id`, so an in-flight row can never match this query. Covered only because the+ ;; shared predicate checks `finished`.+ (testing "an in-flight row contributes nothing"+ (is (not (contains? history in-flight))))+ (testing "a row written before the `finished` column still contributes"+ (is (contains? history legacy))))))))+(deftest message-history-v2-parts-test(testing "stored v2 tool parts are translated to AI-SDK message pairs"(let [conv-id (str (random-uuid))@@ -296,3 +340,50 @@:data_version 2})(is (= requester-id (slackbot.persistence/response-owner-user-id channel-id slack-ts)))(is (= later-user-id (slackbot.persistence/response-owner-user-id channel-id second-slack-ts)))))))))++(deftest state-messages-test+ (let [conv-id (str (random-uuid))+ state-of #(metabot.persistence/conversation-state+ (slackbot.persistence/state-messages conv-id))+ insert! (fn [role state & {:keys [finished error deleted?]+ :or {finished true}}]+ (t2/insert! :model/MetabotMessage+ (cond-> {:conversation_id conv-id+ :role role+ :profile_id "slackbot"+ :total_tokens 0+ :data []+ :data_version 2+ :finished finished+ :state state}+ error (assoc :error error)+ deleted? (assoc :deleted_at (java.time.OffsetDateTime/now)+ :deleted_by_user_id (mt/user->id :rasta)))))]+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})+ (testing "a thread with no turns reconstructs to {}"+ (is (= {} (state-of))))+ (testing "a finished assistant turn's state becomes the baseline"+ (insert! "assistant" {:queries {"q1" {:database 1}} :todos [{:id "a"}]})+ (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}+ (state-of))))+ (testing "user rows are ignored even when they carry state"+ (insert! "user" {:queries {"nope" {:database 99}}} :finished nil)+ (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}+ (state-of))))+ (testing "later turns merge in order -- maps merge entry-wise, vectors take the latest"+ (insert! "assistant" {:queries {"q2" {:database 2}} :todos [{:id "b"}]})+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (state-of))))+ (testing "errored turns never leak into the baseline"+ (insert! "assistant" {:todos [{:id "errored"}]} :error "boom")+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (state-of))))+ (testing "in-flight turns contribute nothing"+ (insert! "assistant" {:todos [{:id "in-flight"}]} :finished nil)+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (state-of))))+ (testing "a response deleted from Slack rewinds its state back out"+ (insert! "assistant" {:todos [{:id "deleted"}]} :deleted? true)+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}+ (state-of)))))))diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex 766caa02111..a44243252b4 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.analytics.prometheus :as prometheus][metabase.channel.slack :as channel.slack]+ [metabase.metabot.agent.core :as agent][metabase.metabot.persistence :as metabot.persistence][metabase.metabot.settings :as metabot.settings][metabase.premium-features.core :as premium-features]@@ -15,7 +16,8 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]- [metabase.util.json :as json]))+ [metabase.util.json :as json]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -89,6 +91,49 @@(is (= :user (:role (first result))))(is (= "Live bot response" (:content (second result))))))))+(deftest thread->history-drops-errored-turns-tool-calls-test+ (testing "an errored turn's tool calls are not replayed, but its Slack text still is"+ (let [conv-id (str (random-uuid))+ clean-ts "1712200000.000002"+ errored-ts "1712200000.000004"+ insert! (fn [slack-ts call-id & {:keys [error]}]+ (t2/insert! :model/MetabotMessage+ (cond-> {:conversation_id conv-id+ :slack_msg_id slack-ts+ :role "assistant"+ :profile_id "slackbot"+ :total_tokens 0+ :data [{:type "tool-search"+ :toolCallId call-id+ :state "output-available"+ :input {:query "orders"}+ :output {:output "<result>orders</result>"}}]+ :data_version 2+ :finished true}+ error (assoc :error error))))]+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})+ ;; The clean row is the control: without it a green assertion cannot tell the filter+ ;; working apart from the fixture never producing tool parts at all.+ (insert! clean-ts "call-clean")+ (insert! errored-ts "call-errored" :error "boom")+ (let [thread {:messages [{:ts "1712200000.000001" :text "First question" :user "U123"}+ {:ts clean-ts :text "Here you go" :bot_id "B123"}+ {:ts "1712200000.000003" :text "Second question" :user "U123"}+ {:ts errored-ts+ :text "Something went wrong. Please try again."+ :bot_id "B123"}]}+ result (#'slackbot.streaming/thread->history thread "UBOT123" conv-id)+ call-ids (into #{} (comp (mapcat :tool_calls) (map :id)) result)]+ (testing "the clean turn's tool call is replayed, the errored turn's is not"+ (is (= #{"call-clean"} call-ids)))+ (testing "both bot messages keep their Slack text -- the thread still shows the failure"+ (is (= ["First question"+ "Here you go"+ "Second question"+ "Something went wrong. Please try again."]+ (into [] (comp (remove #(= :tool (:role %))) (keep :content)) result)))))))))+(deftest format-viz-title-test(testing "format-viz-title builds correct title text"(mt/with-temporary-setting-values [site-url "https://metabase.example.com"]@@ -294,6 +339,69 @@(testing "start-turn! received ai-proxy? = true"(is (=? [{:ai-proxy? true}] @start-opts)))))))))+(deftest slackbot-streaming-seeds-state-from-db-test+ (testing "a turn seeds the agent loop with the state earlier turns in the thread persisted (BOT-522)"+ (tu/with-slackbot-setup+ (let [event-body tu/base-dm-event]+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (tu/with-slackbot-mocks+ {:ai-text "Hello!"}+ (fn [{:keys [ai-request-calls stop-stream-calls]}]+ (letfn [(send! []+ (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body))+ (wait! [n]+ (u/poll {:thunk #(>= (count @stop-stream-calls) n)+ :done? true?+ :timeout-ms 5000}))]+ (send!)+ (wait! 1)+ (testing "the opening turn of a thread starts from an empty baseline"+ (is (= {} (:state (last @ai-request-calls)))))+ ;; Stand in for the state a real first turn would have written: the mocked+ ;; agent loop produces no turn-state of its own. Taking the conversation id+ ;; from the captured opts keeps this independent of how it is derived.+ (t2/insert! :model/MetabotMessage+ {:conversation_id (:conversation-id (last @ai-request-calls))+ :role "assistant"+ :profile_id "slackbot"+ :total_tokens 0+ :data []+ :data_version 2+ :finished true+ :state {:queries {"q1" {:database 1}}}})+ (send!)+ (wait! 2)+ (testing "the next turn in the same thread picks it up instead of {}"+ (is (= {:queries {:q1 {:database 1}}}+ (:state (last @ai-request-calls)))))))))))))++(deftest slackbot-streaming-records-streamed-error-test+ (testing "an :error part the agent loop emits instead of throwing is still recorded on the row,+ so conversation-state does not later replay a failed turn's partial state (BOT-522)"+ (tu/with-slackbot-setup+ (let [event-body tu/base-dm-event+ finalized (promise)]+ (tu/with-slackbot-mocks+ {:ai-text "Hello!"}+ (fn [_ctx]+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop+ (fn [_opts]+ (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (rf init {:type :error :error {:message "boom"}}))))+ metabot.persistence/finalize-assistant-turn!+ (fn [_msg-id _parts & {:as opts}]+ (deliver finalized opts)+ nil)]+ (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body)+ (let [opts (deref finalized 5000 ::timeout)]+ (is (not= ::timeout opts))+ (is (= {:message "boom"} (:error opts)))))))))))+(deftest slackbot-streaming-persists-failed-conversations-test(testing "User row is persisted even if setup throws after it (BOT-1279). With placeholders,start-turn! inserts user + placeholder atomically before any setup runs."