Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-522: Seed the Slack agent loop with persisted conversation state (#80204)

Paolo Rascunà9cd6391ce925
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.clj
index cdc0473eed6..1606e27e493 100644
--- a/src/metabase/metabot/persistence.clj
+++ b/src/metabase/metabot/persistence.clj
@@ -353,7 +353,8 @@
;;; ---------------------------------------- Conversation state ----------------------------------------
-(defn- replayable-assistant-row?
+(defn replayable-assistant-row?
+ "Is `row` an assistant message a later turn may replay?"
[{:keys [error finished] :as row}]
(and (assistant-row? row)
(nil? error)
diff --git a/src/metabase/slackbot/persistence.clj b/src/metabase/slackbot/persistence.clj
index 1e1af931ca0..21b5b99104d 100644
--- a/src/metabase/slackbot/persistence.clj
+++ b/src/metabase/slackbot/persistence.clj
@@ -18,7 +18,8 @@
(into [] (mapcat #(metabot.persistence/tool-part->llm-messages % {:on-unresolved :skip})))))
(defn message-history
- "Tool call history for Slack messages. Returns {slack-msg-id -> [messages...]}."
+ "Tool call history for Slack messages. Returns {slack-msg-id -> [messages...]}.
+ Only [[metabase.metabot.persistence/replayable-assistant-row?]] rows contribute."
[conversation-id slack-msg-ids]
(when (seq slack-msg-ids)
(->> (t2/select :model/MetabotMessage
@@ -26,6 +27,9 @@
:role "assistant"
:deleted_at nil
:slack_msg_id [:in slack-msg-ids])
+ ;; A failed turn's state is dropped by [[metabase.metabot.persistence/conversation-state]], so
+ ;; replaying its tool calls would announce queries the seeded state does not contain.
+ (filter metabot.persistence/replayable-assistant-row?)
(keep (fn [{:keys [slack_msg_id] :as msg}]
(when-let [parts (seq (extract-history-messages msg))]
[slack_msg_id parts])))
@@ -42,6 +46,20 @@
:deleted_at [:not= nil]
:slack_msg_id [:in slack-msg-ids])))
+(defn state-messages
+ "The assistant rows of a Slack thread in reader order, for feeding
+ [[metabase.metabot.persistence/conversation-state]].
+
+ Carries only the columns that function filters and merges on, so a long thread's
+ `data` blobs aren't loaded just to get at `state`. Not usable for history replay —
+ see [[message-history]] for that."
+ [conversation-id]
+ (t2/select [:model/MetabotMessage :id :role :state :error :finished]
+ :conversation_id conversation-id
+ :role "assistant"
+ :deleted_at nil
+ {:order-by [[:created_at :asc] [:id :asc]]}))
+
(defn response-owner-user-id
"Find the Metabase user ID who triggered the assistant response for this Slack channel/message.
Returns nil when the message is not tracked."
diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.clj
index 34deb376915..0f561e94ce9 100644
--- a/src/metabase/slackbot/streaming.clj
+++ b/src/metabase/slackbot/streaming.clj
@@ -222,6 +222,15 @@
:slack-msg-id req-slack-msg-id
:user-id api/*current-user-id*
:ai-proxy? ai-proxy?))
+ ;; Slack replays message text from the thread itself, but the state earlier turns
+ ;; produced only ever existed in the app DB — without it a follow-up cannot resolve
+ ;; an id an earlier turn generated. Mirrors the in-app path in [[metabase.metabot.api]].
+ ;;
+ ;; Deliberately outside the lock's transaction: this turn's own rows are excluded
+ ;; anyway (user row by role, placeholder by NULL `finished`), and a failure here
+ ;; must not roll back the turn we just persisted (BOT-1279).
+ state-messages (slackbot.persistence/state-messages conversation-id)
+ baseline-state (metabot.persistence/conversation-state state-messages)
data-idx (volatile! -1)
request-message (metabot.envelope/user-message (or request-prompt prompt))
capabilities (compute-capabilities)
@@ -276,7 +285,7 @@
(transduce dispatch-xf (constantly nil) nil
(agent/run-agent-loop
{:messages messages
- :state {}
+ :state baseline-state
:profile-id :slackbot
:conversation-id conversation-id
:context context
@@ -294,13 +303,19 @@
;; UPDATE the placeholder with whatever parts we collected, even if the
;; pipeline threw. Raw native parts (not the lossy AI-SDK-message
;; round-trip) preserve tool-output :structured-output for analytics.
- (metabot.persistence/finalize-assistant-turn!
- assistant-msg-id
- (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)
- :profile-id "slackbot"
- :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))
- :turn-state (some-> @memory-atom memory/turn-state)
- :error (some-> @thrown metabot.persistence/throwable->error-payload))))
+ (let [combined-parts (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)]
+ (metabot.persistence/finalize-assistant-turn!
+ assistant-msg-id
+ combined-parts
+ :profile-id "slackbot"
+ :slack-msg-id (when get-res-slack-msg-id (get-res-slack-msg-id))
+ :turn-state (some-> @memory-atom memory/turn-state)
+ ;; A thrown error is more authoritative, but the agent loop catches most
+ ;; failures internally and emits an `:error` part instead of throwing. Without
+ ;; the fallback such a turn persists as a clean `finished` row, and
+ ;; `conversation-state` then merges its partial state into every later turn.
+ :error (or (some-> @thrown metabot.persistence/throwable->error-payload)
+ (:error (u/seek #(= :error (:type %)) combined-parts)))))))
{:msg-id assistant-msg-id
:external-id assistant-external-id}))
diff --git a/test/metabase/slackbot/persistence_test.clj b/test/metabase/slackbot/persistence_test.clj
index c22e690de6e..2a234f99572 100644
--- a/test/metabase/slackbot/persistence_test.clj
+++ b/test/metabase/slackbot/persistence_test.clj
@@ -1,6 +1,7 @@
(ns metabase.slackbot.persistence-test
(:require
[clojure.test :refer :all]
+ [metabase.metabot.persistence :as metabot.persistence]
[metabase.slackbot.persistence :as slackbot.persistence]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -79,6 +80,49 @@
(is (= #{deleted-ts}
(slackbot.persistence/deleted-message-ids conv-id #{deleted-ts}))))))))
+(deftest message-history-excludes-non-replayable-rows-test
+ (testing "only rows a later turn may replay contribute tool history"
+ (let [conv-id (str (random-uuid))
+ insert! (fn [slack-ts call-id & {:keys [error] :as opts}]
+ (t2/insert! :model/MetabotMessage
+ (cond-> {:conversation_id conv-id
+ :slack_msg_id slack-ts
+ :role "assistant"
+ :profile_id "slackbot"
+ :total_tokens 0
+ :data [{:type "tool-search"
+ :toolCallId call-id
+ :state "output-available"
+ :input {}
+ :output {:output "result"}}]
+ :data_version 2}
+ (contains? opts :finished) (assoc :finished (:finished opts))
+ error (assoc :error error))))
+ clean "1712100000.000001"
+ errored "1712100000.000002"
+ in-flight "1712100000.000003"
+ legacy "1712100000.000004"]
+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]
+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})
+ (insert! clean "call-clean" :finished true)
+ (insert! errored "call-errored" :finished true :error "boom")
+ (insert! in-flight "call-in-flight" :finished nil)
+ ;; No :finished at all -- exercises the column default that keeps pre-migration threads readable.
+ (insert! legacy "call-legacy")
+ (let [history (slackbot.persistence/message-history
+ conv-id #{clean errored in-flight legacy})]
+ (testing "a clean finished row still contributes -- guards against over-filtering"
+ (is (contains? history clean)))
+ (testing "an errored row contributes nothing"
+ (is (not (contains? history errored))))
+ ;; Unreachable in production: insert-assistant-placeholder! writes neither `finished` nor
+ ;; `slack_msg_id`, so an in-flight row can never match this query. Covered only because the
+ ;; shared predicate checks `finished`.
+ (testing "an in-flight row contributes nothing"
+ (is (not (contains? history in-flight))))
+ (testing "a row written before the `finished` column still contributes"
+ (is (contains? history legacy))))))))
+
(deftest message-history-v2-parts-test
(testing "stored v2 tool parts are translated to AI-SDK message pairs"
(let [conv-id (str (random-uuid))
@@ -296,3 +340,50 @@
:data_version 2})
(is (= requester-id (slackbot.persistence/response-owner-user-id channel-id slack-ts)))
(is (= later-user-id (slackbot.persistence/response-owner-user-id channel-id second-slack-ts)))))))))
+
+(deftest state-messages-test
+ (let [conv-id (str (random-uuid))
+ state-of #(metabot.persistence/conversation-state
+ (slackbot.persistence/state-messages conv-id))
+ insert! (fn [role state & {:keys [finished error deleted?]
+ :or {finished true}}]
+ (t2/insert! :model/MetabotMessage
+ (cond-> {:conversation_id conv-id
+ :role role
+ :profile_id "slackbot"
+ :total_tokens 0
+ :data []
+ :data_version 2
+ :finished finished
+ :state state}
+ error (assoc :error error)
+ deleted? (assoc :deleted_at (java.time.OffsetDateTime/now)
+ :deleted_by_user_id (mt/user->id :rasta)))))]
+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]
+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})
+ (testing "a thread with no turns reconstructs to {}"
+ (is (= {} (state-of))))
+ (testing "a finished assistant turn's state becomes the baseline"
+ (insert! "assistant" {:queries {"q1" {:database 1}} :todos [{:id "a"}]})
+ (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}
+ (state-of))))
+ (testing "user rows are ignored even when they carry state"
+ (insert! "user" {:queries {"nope" {:database 99}}} :finished nil)
+ (is (= {:queries {:q1 {:database 1}} :todos [{:id "a"}]}
+ (state-of))))
+ (testing "later turns merge in order -- maps merge entry-wise, vectors take the latest"
+ (insert! "assistant" {:queries {"q2" {:database 2}} :todos [{:id "b"}]})
+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}
+ (state-of))))
+ (testing "errored turns never leak into the baseline"
+ (insert! "assistant" {:todos [{:id "errored"}]} :error "boom")
+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}
+ (state-of))))
+ (testing "in-flight turns contribute nothing"
+ (insert! "assistant" {:todos [{:id "in-flight"}]} :finished nil)
+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}
+ (state-of))))
+ (testing "a response deleted from Slack rewinds its state back out"
+ (insert! "assistant" {:todos [{:id "deleted"}]} :deleted? true)
+ (is (= {:queries {:q1 {:database 1} :q2 {:database 2}} :todos [{:id "b"}]}
+ (state-of)))))))
diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.clj
index 766caa02111..a44243252b4 100644
--- a/test/metabase/slackbot/streaming_test.clj
+++ b/test/metabase/slackbot/streaming_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[metabase.analytics.prometheus :as prometheus]
[metabase.channel.slack :as channel.slack]
+ [metabase.metabot.agent.core :as agent]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.metabot.settings :as metabot.settings]
[metabase.premium-features.core :as premium-features]
@@ -15,7 +16,8 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
- [metabase.util.json :as json]))
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -89,6 +91,49 @@
(is (= :user (:role (first result))))
(is (= "Live bot response" (:content (second result))))))))
+(deftest thread->history-drops-errored-turns-tool-calls-test
+ (testing "an errored turn's tool calls are not replayed, but its Slack text still is"
+ (let [conv-id (str (random-uuid))
+ clean-ts "1712200000.000002"
+ errored-ts "1712200000.000004"
+ insert! (fn [slack-ts call-id & {:keys [error]}]
+ (t2/insert! :model/MetabotMessage
+ (cond-> {:conversation_id conv-id
+ :slack_msg_id slack-ts
+ :role "assistant"
+ :profile_id "slackbot"
+ :total_tokens 0
+ :data [{:type "tool-search"
+ :toolCallId call-id
+ :state "output-available"
+ :input {:query "orders"}
+ :output {:output "<result>orders</result>"}}]
+ :data_version 2
+ :finished true}
+ error (assoc :error error))))]
+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]
+ (t2/insert! :model/MetabotConversation {:id conv-id :user_id (mt/user->id :rasta)})
+ ;; The clean row is the control: without it a green assertion cannot tell the filter
+ ;; working apart from the fixture never producing tool parts at all.
+ (insert! clean-ts "call-clean")
+ (insert! errored-ts "call-errored" :error "boom")
+ (let [thread {:messages [{:ts "1712200000.000001" :text "First question" :user "U123"}
+ {:ts clean-ts :text "Here you go" :bot_id "B123"}
+ {:ts "1712200000.000003" :text "Second question" :user "U123"}
+ {:ts errored-ts
+ :text "Something went wrong. Please try again."
+ :bot_id "B123"}]}
+ result (#'slackbot.streaming/thread->history thread "UBOT123" conv-id)
+ call-ids (into #{} (comp (mapcat :tool_calls) (map :id)) result)]
+ (testing "the clean turn's tool call is replayed, the errored turn's is not"
+ (is (= #{"call-clean"} call-ids)))
+ (testing "both bot messages keep their Slack text -- the thread still shows the failure"
+ (is (= ["First question"
+ "Here you go"
+ "Second question"
+ "Something went wrong. Please try again."]
+ (into [] (comp (remove #(= :tool (:role %))) (keep :content)) result)))))))))
+
(deftest format-viz-title-test
(testing "format-viz-title builds correct title text"
(mt/with-temporary-setting-values [site-url "https://metabase.example.com"]
@@ -294,6 +339,69 @@
(testing "start-turn! received ai-proxy? = true"
(is (=? [{:ai-proxy? true}] @start-opts)))))))))
+(deftest slackbot-streaming-seeds-state-from-db-test
+ (testing "a turn seeds the agent loop with the state earlier turns in the thread persisted (BOT-522)"
+ (tu/with-slackbot-setup
+ (let [event-body tu/base-dm-event]
+ (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]
+ (tu/with-slackbot-mocks
+ {:ai-text "Hello!"}
+ (fn [{:keys [ai-request-calls stop-stream-calls]}]
+ (letfn [(send! []
+ (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body))
+ (wait! [n]
+ (u/poll {:thunk #(>= (count @stop-stream-calls) n)
+ :done? true?
+ :timeout-ms 5000}))]
+ (send!)
+ (wait! 1)
+ (testing "the opening turn of a thread starts from an empty baseline"
+ (is (= {} (:state (last @ai-request-calls)))))
+ ;; Stand in for the state a real first turn would have written: the mocked
+ ;; agent loop produces no turn-state of its own. Taking the conversation id
+ ;; from the captured opts keeps this independent of how it is derived.
+ (t2/insert! :model/MetabotMessage
+ {:conversation_id (:conversation-id (last @ai-request-calls))
+ :role "assistant"
+ :profile_id "slackbot"
+ :total_tokens 0
+ :data []
+ :data_version 2
+ :finished true
+ :state {:queries {"q1" {:database 1}}}})
+ (send!)
+ (wait! 2)
+ (testing "the next turn in the same thread picks it up instead of {}"
+ (is (= {:queries {:q1 {:database 1}}}
+ (:state (last @ai-request-calls)))))))))))))
+
+(deftest slackbot-streaming-records-streamed-error-test
+ (testing "an :error part the agent loop emits instead of throwing is still recorded on the row,
+ so conversation-state does not later replay a failed turn's partial state (BOT-522)"
+ (tu/with-slackbot-setup
+ (let [event-body tu/base-dm-event
+ finalized (promise)]
+ (tu/with-slackbot-mocks
+ {:ai-text "Hello!"}
+ (fn [_ctx]
+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop
+ (fn [_opts]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (rf init {:type :error :error {:message "boom"}}))))
+ metabot.persistence/finalize-assistant-turn!
+ (fn [_msg-id _parts & {:as opts}]
+ (deliver finalized opts)
+ nil)]
+ (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)
+ (let [opts (deref finalized 5000 ::timeout)]
+ (is (not= ::timeout opts))
+ (is (= {:message "boom"} (:error opts)))))))))))
+
(deftest slackbot-streaming-persists-failed-conversations-test
(testing "User row is persisted even if setup throws after it (BOT-1279). With placeholders,
start-turn! inserts user + placeholder atomically before any setup runs."