Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1606: Truncate over-long Metabot answers instead of going silent (#80105)

Paolo Rascunà638d0e4cb28b
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index aaf9a97bcf3..6a9ceb38ad2 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -732,6 +732,15 @@
(prometheus/counter :metabase-slackbot/file-uploads
{:description "Number of file uploads via the Slack bot."
:labels [:result]})
+ (prometheus/counter :metabase-slackbot/responses-truncated
+ {:description (str "Number of Slack bot responses truncated because they exceeded "
+ "Slack's message limits.")})
+ (prometheus/counter :metabase-slackbot/viz-links-dropped
+ {:description (str "Number of visualization query links dropped from Slack bot messages "
+ "because the linked title exceeded Slack's block limits.")})
+ (prometheus/counter :metabase-slackbot/responses-undeliverable
+ {:description (str "Number of Slack bot responses that reached the user as nothing at all, "
+ "because the plain-text fallback failed too.")})
;; metabot / LLM agent metrics
(prometheus/counter :metabase-metabot/llm-requests
{:description "LLM provider API requests"
diff --git a/src/metabase/slackbot/channel.clj b/src/metabase/slackbot/channel.clj
index 0e3dd28bd00..8b4556b846f 100644
--- a/src/metabase/slackbot/channel.clj
+++ b/src/metabase/slackbot/channel.clj
@@ -2,9 +2,12 @@
"Visible Slack channel reply/update flow for metabot."
(:require
[clojure.string :as str]
+ [metabase.analytics-interface.core :as analytics]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.slackbot.client :as slackbot.client]
- [metabase.util.log :as log]))
+ [metabase.system.core :as system]
+ [metabase.util.log :as log]
+ [metabase.util.string :as u.str]))
(set! *warn-on-reflection* true)
@@ -20,6 +23,12 @@
[prompt]
(str prompt channel-response-style-suffix))
+(def section-text-limit
+ "Slack rejects a `section` block whose `text.text` exceeds this many characters.
+ The whole `chat.postMessage` fails with `invalid_blocks`.
+ See https://docs.slack.dev/reference/block-kit/blocks/section-block."
+ 3000)
+
(defn- final-text-blocks
"Build the leading text block(s) for a finalized non-streaming Slack message."
[text]
@@ -29,6 +38,30 @@
:text {:type "mrkdwn"
:text text}}]))
+(defn- truncation-notice
+ "Copy explaining that an answer was cut short.
+ `url` points at the Metabase instance; nil leaves the sentence standing without a link."
+ [url]
+ (str "_This answer was longer than " section-text-limit " characters. That is too long to post in "
+ "Slack, so I cut it short._\n\n"
+ "Ask a narrower question so the answer comes back smaller"
+ (if url
+ (str ", or head to <" url "|Metabase> and try again.")
+ ".")))
+
+(defn- truncation-block
+ "A muted aside explaining a cut answer, for the same message the answer was posted in."
+ []
+ ;; `context` rather than a second section: Slack renders it small and grey, so it reads as an
+ ;; aside about the answer rather than as part of it.
+ ;;
+ ;; The instance home page, not the conversation that produced the answer: picking a Slack thread
+ ;; back up on the web is not supported, so a deep link would only lead somewhere that cannot help.
+ ;; `site-url` is guaranteed to carry no trailing slash, so it stands as a link on its own.
+ {:type "context"
+ :elements [{:type "mrkdwn"
+ :text (truncation-notice (system/site-url))}]})
+
(defn- make-channel-callbacks
"Create callback functions for channel replies.
Uses the Slack assistant setStatus API for progress indication.
@@ -87,13 +120,26 @@
(set-status! "Rendering results..."))
(let [{:keys [blocks errors]} (collect-viz-blocks @prefetched-viz)
answer-text (str/trim @current-text)
- final-text (if (or (seq answer-text) (seq blocks))
+ answer (if (or (seq answer-text) (seq blocks))
answer-text
"I wasn't able to generate a response. Please try again.")
- final-blocks (into (into (final-text-blocks final-text) blocks)
- (feedback-blocks conversation-id message-external-id))
+ ;; `final-text` doubles as the message's `:text` -- Slack's notification preview, and
+ ;; what `streaming/thread->history` replays to the model as the assistant's own words.
+ ;; The notice stays out of it, so the model is never told it said this.
+ final-text (u.str/elide answer section-text-limit)
+ ;; Derived rather than re-tested: `elide` returns `answer` itself when it fits, so
+ ;; the notice cannot disagree with whether a cut actually happened.
+ truncated? (not= final-text answer)
+ final-blocks (-> (final-text-blocks final-text)
+ (cond-> truncated? (conj (truncation-block)))
+ (into blocks)
+ (into (feedback-blocks conversation-id message-external-id)))
res (slackbot.client/post-thread-reply client {:channel channel :thread_ts thread-ts}
final-text :blocks final-blocks)]
+ (when truncated?
+ (log/infof "[slackbot] channel answer truncated (answer_length=%d limit=%d)"
+ (count answer) section-text-limit)
+ (analytics/inc! :metabase-slackbot/responses-truncated))
(when-let [res-ts (:ts res)]
(metabot.persistence/set-response-slack-msg-id! assistant-msg-id res-ts))
(when-not (:ok res)
@@ -101,7 +147,16 @@
(:error res)
(count final-blocks)
(pr-str (mapv :type final-blocks))
- (pr-str (get-in res [:response_metadata :messages]))))
+ (pr-str (get-in res [:response_metadata :messages])))
+ ;; The blocks were rejected for a reason we did not anticipate. A plain-text reply
+ ;; carries no blocks for Slack to reject, so the user gets something back rather than
+ ;; silence (BOT-1606).
+ (let [fallback (slackbot.client/post-thread-reply
+ client message-ctx
+ "I generated a response, but Slack could not render it. Please try again.")]
+ (when-not (:ok fallback)
+ (log/errorf "[slackbot] channel fallback post-message failed: %s" (:error fallback))
+ (analytics/inc! :metabase-slackbot/responses-undeliverable))))
(doseq [e errors]
(post-viz-error! client channel thread-ts e))))
(catch Exception e
diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.clj
index 97a177b41bf..34deb376915 100644
--- a/src/metabase/slackbot/streaming.clj
+++ b/src/metabase/slackbot/streaming.clj
@@ -4,6 +4,7 @@
(:require
[clojure.string :as str]
[java-time.api :as t]
+ [metabase.analytics-interface.core :as analytics]
[metabase.api.common :as api]
[metabase.channel.slack :as channel.slack]
[metabase.llm.provider :as llm.provider]
@@ -24,7 +25,8 @@
[metabase.system.core :as system]
[metabase.util :as u]
[metabase.util.json :as json]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [metabase.util.string :as u.str])
(:import
(java.util.concurrent
Callable
@@ -113,16 +115,41 @@
(str/replace ">" "&gt;")
(str/replace "|" "\u2502")))
+(def ^:private image-alt-text-limit
+ "Slack rejects an `image` block whose `alt_text` exceeds this many characters.
+ Tighter than [[slackbot.channel/section-text-limit]], so a title can fit the section and still
+ fail here.
+ See https://docs.slack.dev/reference/block-kit/blocks/image-block."
+ 2000)
+
(defn- format-viz-title
- "Build the title text for a visualization message.
- Combines the title with a link to the query in Metabase."
+ "Build the title text for a visualization message, combining the title with a link to the query.
+ Returns nil when nothing is left to show, and never exceeds
+ [[slackbot.channel/section-text-limit]]."
[title link]
- (let [full-link (when link (str (system/site-url) link))]
- (cond
- (and title full-link) (str "\ud83d\udcca <" full-link "|" (escape-slack-link-text title) ">")
- title title
- full-link (str "\ud83d\udcca <" full-link "|Open in Metabase>")
- :else nil)))
+ (let [limit slackbot.channel/section-text-limit
+ full-link (when link (str (system/site-url) link))
+ text (cond
+ (and title full-link) (str "\ud83d\udcca <" full-link "|" (escape-slack-link-text title) ">")
+ title title
+ full-link (str "\ud83d\udcca <" full-link "|Open in Metabase>")
+ :else nil)]
+ ;; An adhoc `/question#<base64-query>` link can run past the section limit on its own, and Slack
+ ;; then rejects the whole message (BOT-1606). A cut URL is a broken URL, so drop the link rather
+ ;; than truncate it; with no title left, `viz-output->blocks` falls back to "Query results".
+ (if (and text (> (count text) limit))
+ (do
+ ;; Either way the answer is the same elided title -- only what we report differs, because
+ ;; with no link there is nothing to drop and the counter would overstate what happened.
+ (if full-link
+ (do
+ (log/infof "[slackbot] viz title over limit, dropping query link (text_length=%d link_length=%d limit=%d)"
+ (count text) (count full-link) limit)
+ (analytics/inc! :metabase-slackbot/viz-links-dropped))
+ (log/infof "[slackbot] viz title over limit with no link to drop, eliding it (title_length=%d limit=%d)"
+ (count text) limit))
+ (u.str/elide title limit))
+ text)))
(defn- viz-output->blocks
"Build blocks for a visualization to be included in the finalized stop-stream message."
@@ -138,7 +165,10 @@
:text {:type "mrkdwn" :text text}}
{:type "image"
:slack_file {:id id}
- :alt_text (or title filename "Visualization")}]]
+ ;; Plain text, so it is cut rather than elided -- no ellipsis to add.
+ :alt_text (or (some-> title (u.str/limit-chars image-alt-text-limit))
+ filename
+ "Visualization")}]]
blocks))))
(def ^:private tool-friendly-names
@@ -604,7 +634,9 @@
message-ctx
"I generated a response, but Slack could not render it. Please try again.")]
(when-not (:ok fallback-result)
- (log/errorf "[slackbot] fallback post-message failed after stop-stream error: %s" (:error fallback-result)))))
+ (log/errorf "[slackbot] fallback post-message failed after stop-stream error: %s"
+ (:error fallback-result))
+ (analytics/inc! :metabase-slackbot/responses-undeliverable))))
(doseq [e errors]
(post-viz-error! client channel thread-ts e)))
(slackbot.client/post-thread-reply client message-ctx "I wasn't able to generate a response. Please try again.")))
diff --git a/test/metabase/slackbot/api_test.clj b/test/metabase/slackbot/api_test.clj
index cf756944f96..33bfbff4cab 100644
--- a/test/metabase/slackbot/api_test.clj
+++ b/test/metabase/slackbot/api_test.clj
@@ -206,6 +206,39 @@
(let [msg (t2/select-one :model/MetabotMessage :channel_id channel-id :role "assistant")]
(is (some? (:slack_msg_id msg))))))))))))
+(deftest app-mention-long-answer-fits-slack-blocks-test
+ (testing "POST /events with app_mention truncates a long answer so Slack accepts it (BOT-1606)"
+ (tu/with-slackbot-setup
+ (let [channel-id "C-LONG-ANSWER-TEST"
+ event-body (assoc-in tu/base-mention-event [:event :channel] channel-id)]
+ (tu/with-slackbot-mocks
+ {:ai-text tu/oversized-answer}
+ (fn [{:keys [post-calls]}]
+ (let [response (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)]
+ (is (= "ok" response))
+ (u/poll {:thunk #(t2/select-one :model/MetabotMessage
+ :channel_id channel-id :role "assistant"
+ :slack_msg_id [:not= nil])
+ :done? some?
+ :timeout-ms 5000})
+ (is (= 1 (count @post-calls)))
+ (let [blocks (:blocks (first @post-calls))]
+ (testing "no section block is over the limit -- unlike the untruncated answer"
+ (is (nil? (tu/oversized-section-error blocks)))
+ (is (some? (tu/oversized-section-error [{:type "section"
+ :text {:type "mrkdwn" :text tu/oversized-answer}}]))
+ "the answer really is past the limit, so the case under test is the real one"))
+ (testing "the answer is cut to the limit, and the message says why"
+ (is (= tu/slack-section-text-limit
+ (count (get-in (first blocks) [:text :text]))))
+ (is (some (fn [block]
+ (and (= "context" (:type block))
+ (str/includes? (get-in block [:elements 0 :text])
+ "too long to post in Slack")))
+ blocks)))))))))))
+
(deftest stream-start-failure-test
(testing "When start-stream fails, falls back to a regular message"
(tu/with-slackbot-setup
@@ -229,6 +262,29 @@
(testing "stop-stream is never called"
(is (= 0 (count @stop-stream-calls))))))))))))
+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.
+(deftest dm-response-undeliverable-metric-test
+ (testing "a DM whose stream and plain-text fallback both fail is counted as undeliverable"
+ (tu/with-slackbot-setup
+ (mt/with-prometheus-system! [_ system]
+ (let [event-body tu/base-dm-event]
+ (tu/with-slackbot-mocks
+ {:ai-text "Here is your answer"}
+ (fn [_]
+ ;; `post-thread-reply` delegates to `post-message`, so failing that fails the
+ ;; fallback too -- the user ends up with nothing at all.
+ (mt/with-dynamic-fn-redefs
+ [slackbot.client/stop-stream (constantly {:ok false :error "invalid_blocks"})
+ slackbot.client/post-message (constantly {:ok false :error "channel_not_found"})]
+ (let [response (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)]
+ (is (= "ok" response))
+ (u/poll {:thunk #(mt/metric-value system :metabase-slackbot/responses-undeliverable)
+ :done? pos?
+ :timeout-ms 5000})
+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))))))))))
+
(deftest ai-request-error-stops-stream-test
(testing "When the agent loop throws after the stream has started, the stream is stopped"
(tu/with-slackbot-setup
diff --git a/test/metabase/slackbot/channel_test.clj b/test/metabase/slackbot/channel_test.clj
index 8b1fe6db696..7736829ff00 100644
--- a/test/metabase/slackbot/channel_test.clj
+++ b/test/metabase/slackbot/channel_test.clj
@@ -1,9 +1,12 @@
(ns metabase.slackbot.channel-test
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.analytics.prometheus :as prometheus]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.slackbot.channel :as slackbot.channel]
[metabase.slackbot.client :as slackbot.client]
+ [metabase.slackbot.test-util :as slackbot.tu]
[metabase.test :as mt]))
(deftest channel-response-passes-slack-metadata-for-deep-linking-test
@@ -43,3 +46,182 @@
(select-keys @request-opts [:team-id :thread-ts :req-slack-msg-id])))
(is (= {:msg-id 42 :slack-msg-id "1700000000.000002"}
@backfill-args))))
+
+;;; ---------------------------------- BOT-1606: over-long answers ----------------------------------
+
+(def ^:private conversation-id "0f8a1c3e-0000-4000-8000-000000000001")
+
+(def ^:private test-site-url "https://metabase.example.com")
+
+(def ^:private viz-blocks
+ [{:type "section" :text {:type "mrkdwn" :text "*Orders by month*"}}
+ {:type "table" :rows []}])
+
+(def ^:private feedback-blocks
+ [{:type "context_actions" :elements []}])
+
+(defn- drive-channel-response!
+ "Drive [[slackbot.channel/send-channel-response]] to completion with `answer` as the model's text.
+ `reply-result` stands in for what Slack returns from `post-thread-reply`, and is called with the
+ number of posts made so far. Returns `{:posts [{:text .. :blocks ..} ..] :backfill ..}`, in the
+ order the posts were made."
+ [answer reply-result]
+ (let [posts (atom [])
+ backfill (atom nil)]
+ (mt/with-dynamic-fn-redefs [slackbot.client/set-status (constantly {:ok true})
+ slackbot.client/post-thread-reply
+ (fn [_client _message-ctx text & {:keys [blocks]}]
+ (reply-result (count (swap! posts conj {:text text :blocks blocks}))))
+ metabot.persistence/set-response-slack-msg-id!
+ (fn [msg-id slack-msg-id]
+ (reset! backfill {:msg-id msg-id :slack-msg-id slack-msg-id}))]
+ (slackbot.channel/send-channel-response
+ {}
+ {:ts "1700000000.000001"}
+ nil
+ {:channel-id "C123"
+ :message-ctx {:channel "C123" :thread_ts "1700000000.000001"}
+ :channel "C123"
+ :thread-ts "1700000000.000001"
+ :auth-info {:team_id "T123"}
+ :thread {:messages []}
+ :bot-user-id "U999"
+ :prompt "hello"
+ :conversation-id conversation-id}
+ {:tool-name->friendly {}
+ :make-streaming-ai-request (fn [& args]
+ ((:on-text (last args)) answer)
+ {:msg-id 42
+ :external-id "message-external-id"})
+ :collect-viz-blocks (constantly {:blocks viz-blocks :errors []})
+ :feedback-blocks (constantly feedback-blocks)
+ :post-viz-error! (constantly nil)
+ :make-viz-prefetch-callback (constantly (fn [& _]))
+ :cancel-prefetched-viz! (constantly nil)}))
+ {:posts @posts :backfill @backfill}))
+
+(defn- send-channel-response!
+ "Drive [[slackbot.channel/send-channel-response]] with a Slack that accepts everything.
+ Returns `{:text .. :blocks ..}` as posted, plus the `:backfill` it recorded."
+ [answer]
+ (let [{:keys [posts backfill]} (drive-channel-response!
+ answer
+ (constantly {:ok true :ts "1700000000.000002"}))]
+ (assoc (last posts) :backfill backfill)))
+
+(deftest ^:parallel truncation-notice-test
+ (testing "the notice points at the instance, and says what to do about the cut"
+ (is (= (str "_This answer was longer than 3000 characters. That is too long to post in Slack, "
+ "so I cut it short._\n\n"
+ "Ask a narrower question so the answer comes back smaller"
+ ", or head to <https://metabase.example.com|Metabase> and try again.")
+ (#'slackbot.channel/truncation-notice test-site-url))))
+ (testing "an instance with no site URL still gets a sentence, just without the link"
+ (is (= (str "_This answer was longer than 3000 characters. That is too long to post in Slack, "
+ "so I cut it short._\n\n"
+ "Ask a narrower question so the answer comes back smaller.")
+ (#'slackbot.channel/truncation-notice nil))))
+ (testing "the notice never deep-links to the conversation -- continuing it on the web is unsupported"
+ (is (not (str/includes? (#'slackbot.channel/truncation-notice test-site-url)
+ "/metabot/conversation/")))))
+
+(deftest channel-response-truncates-oversized-answer-test
+ (testing "an oversized answer is cut, and the same message explains why (BOT-1606)"
+ (mt/with-temporary-setting-values [site-url test-site-url]
+ (let [{:keys [text blocks backfill]} (send-channel-response! slackbot.tu/oversized-answer)
+ [answer-block notice-block] blocks
+ section-text (get-in answer-block [:text :text])]
+ (testing "no section block is over the limit -- so Slack no longer rejects the message"
+ (is (nil? (slackbot.tu/oversized-section-error blocks))))
+ (testing "the answer rides one mrkdwn section block, cut to the limit"
+ (is (= "section" (:type answer-block)))
+ (is (= "mrkdwn" (get-in answer-block [:text :type])))
+ (is (= slackbot.tu/slack-section-text-limit (count section-text)))
+ (is (str/ends-with? section-text "..."))
+ (is (str/starts-with? slackbot.tu/oversized-answer (str/replace section-text #"\.\.\.$" ""))
+ "what survives the cut is a genuine prefix of the answer"))
+ (testing "the notice rides in a context block of the same message, linking to the instance"
+ (is (= "context" (:type notice-block)))
+ (is (= [{:type "mrkdwn"
+ :text (#'slackbot.channel/truncation-notice test-site-url)}]
+ (:elements notice-block)))
+ (is (not (str/includes? (get-in notice-block [:elements 0 :text]) "/metabot/conversation/"))
+ "the link is the home page, not a conversation we cannot continue on the web"))
+ (testing "the visualizations and the feedback buttons still ride the message"
+ (is (= ["section" "context" "section" "table" "context_actions"]
+ (mapv :type blocks))))
+ (testing "the notice stays out of `:text`, which `thread->history` replays back to the model"
+ (is (= section-text text))
+ (is (not (str/includes? text "too long to post in Slack"))))
+ (testing "one message is posted, and its ts is persisted -- it carries the feedback buttons"
+ (is (= {:msg-id 42 :slack-msg-id "1700000000.000002"} backfill)))))))
+
+(deftest channel-response-truncation-notice-without-site-url-test
+ (testing "an instance with no site URL still gets the notice, just without a link"
+ (mt/with-temporary-setting-values [site-url nil]
+ (let [{:keys [blocks]} (send-channel-response! slackbot.tu/oversized-answer)
+ notice-block (second blocks)
+ notice (get-in notice-block [:elements 0 :text])]
+ (is (= "context" (:type notice-block)))
+ (is (str/includes? notice "too long to post in Slack"))
+ (is (not (str/includes? notice "<"))
+ "nothing to link to, so the sentence stands on its own")))))
+
+(deftest channel-response-does-not-truncate-when-it-fits-test
+ (testing "an answer within the limit is posted whole, with no notice"
+ (let [answer "Orders peaked in March. *1,204* of them."
+ {:keys [text blocks]} (send-channel-response! answer)]
+ (is (nil? (slackbot.tu/oversized-section-error blocks)))
+ (is (= answer (get-in (first blocks) [:text :text])))
+ (is (= answer text))
+ (is (= ["section" "section" "table" "context_actions"] (mapv :type blocks))
+ "no context block, because nothing was cut"))))
+
+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.
+(deftest channel-response-truncation-metric-test
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-temporary-setting-values [site-url test-site-url]
+ (testing "truncating an answer increments the truncation counter"
+ (send-channel-response! slackbot.tu/oversized-answer)
+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-truncated))))
+ (testing "an answer that fits does not"
+ (prometheus/clear! :metabase-slackbot/responses-truncated)
+ (send-channel-response! "Short answer.")
+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-truncated)))))))
+
+(deftest channel-response-falls-back-to-plain-text-when-blocks-are-rejected-test
+ (testing "a message Slack rejects still reaches the user as plain text, rather than silence"
+ (let [{:keys [posts]} (drive-channel-response!
+ "Orders peaked in March."
+ (fn [n] (if (= n 1)
+ {:ok false :error "invalid_blocks"}
+ {:ok true :ts "1700000000.000002"})))
+ [rejected fallback] posts]
+ (is (= 2 (count posts))
+ "the rejected post is followed by a plain-text retry")
+ (is (seq (:blocks rejected))
+ "the first attempt carried blocks")
+ (is (nil? (:blocks fallback))
+ "the fallback carries none, so there is nothing left for Slack to reject")
+ (is (= "I generated a response, but Slack could not render it. Please try again."
+ (:text fallback)))))
+ (testing "a message Slack accepts is posted once, with no fallback"
+ (let [{:keys [posts]} (drive-channel-response!
+ "Orders peaked in March."
+ (constantly {:ok true :ts "1700000000.000002"}))]
+ (is (= 1 (count posts))))))
+
+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.
+(deftest channel-response-undeliverable-metric-test
+ (mt/with-prometheus-system! [_ system]
+ (testing "a response that fails even as plain text is counted as undeliverable"
+ (drive-channel-response! "Orders peaked in March."
+ (constantly {:ok false :error "channel_not_found"}))
+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))
+ (testing "a response saved by the fallback is not -- the user did get a reply"
+ (prometheus/clear! :metabase-slackbot/responses-undeliverable)
+ (drive-channel-response! "Orders peaked in March."
+ (fn [n] (if (= n 1)
+ {:ok false :error "invalid_blocks"}
+ {:ok true :ts "1700000000.000002"})))
+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))))
diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.clj
index fe4abdd1ff0..766caa02111 100644
--- a/test/metabase/slackbot/streaming_test.clj
+++ b/test/metabase/slackbot/streaming_test.clj
@@ -1,6 +1,9 @@
(ns metabase.slackbot.streaming-test
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.channel.slack :as channel.slack]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.metabot.settings :as metabot.settings]
[metabase.premium-features.core :as premium-features]
@@ -107,7 +110,97 @@
(#'slackbot.streaming/format-viz-title "Foo <Bar> | Baz" "/question/42"))))
(testing "title-only does not escape (no link syntax)"
(is (= "Sales & Revenue"
- (#'slackbot.streaming/format-viz-title "Sales & Revenue" nil)))))))
+ (#'slackbot.streaming/format-viz-title "Sales & Revenue" nil))))
+ (testing "an over-long link is dropped, not cut -- a truncated URL is a broken URL (BOT-1606)"
+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))]
+ (is (= "My Chart"
+ (#'slackbot.streaming/format-viz-title "My Chart" huge-link))
+ "the title survives; the link that cannot fit does not")
+ (is (nil? (#'slackbot.streaming/format-viz-title nil huge-link))
+ "nothing left to keep, so viz-output->blocks falls back to Query results"))))))
+
+(deftest viz-blocks-fit-slack-section-limit-test
+ (testing "a viz whose query link runs past the section limit still posts (BOT-1606)"
+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]
+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))
+ blocks (#'slackbot.streaming/viz-output->blocks
+ {:type :table :content [{:type "table" :rows []}]}
+ "revenue-by-month"
+ "Revenue by month"
+ huge-link)]
+ (is (nil? (tu/oversized-section-error blocks))
+ "Slack no longer rejects the whole message")
+ (is (some? (tu/oversized-section-error
+ [{:type "section"
+ :text {:type "mrkdwn"
+ :text (str "📊 <https://metabase.example.com" huge-link "|Revenue by month>")}}]))
+ "the linked title really is past the limit, so the case under test is the real one")
+ (is (= "Revenue by month" (get-in (first blocks) [:text :text])))
+ (is (= ["section" "table"] (mapv :type blocks)))))))
+
+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.
+(deftest viz-title-over-limit-is-observable-test
+ (testing "dropping an over-long query link is logged and counted (BOT-1606)"
+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]
+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))]
+ (testing "the drop is recorded in the log"
+ ;; `log/infof` only evaluates its args once the level is enabled, so without this the
+ ;; line never runs under test and a bad format arg would ship unnoticed.
+ (mt/with-log-messages-for-level [messages [metabase.slackbot.streaming :info]]
+ (#'slackbot.streaming/format-viz-title "Revenue by month" huge-link)
+ (is (some (fn [{:keys [message]}]
+ (str/includes? message "viz title over limit, dropping query link"))
+ (messages)))))
+ (testing "and counted on its own metric, separate from answer truncation"
+ (mt/with-prometheus-system! [_ system]
+ (#'slackbot.streaming/format-viz-title "Revenue by month" huge-link)
+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped)))
+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-truncated))
+ "an over-long link is not an over-long answer")
+ (testing "a link that fits is not counted"
+ (prometheus/clear! :metabase-slackbot/viz-links-dropped)
+ (#'slackbot.streaming/format-viz-title "Revenue by month" "/question/42")
+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped))))))))))
+
+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.
+(deftest viz-title-over-limit-without-link-is-not-a-dropped-link-test
+ (testing "an over-long title with no link to drop is elided, but not counted as a dropped link"
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]
+ (let [long-title (apply str (repeat 4000 "T"))]
+ (is (= tu/slack-section-text-limit
+ (count (#'slackbot.streaming/format-viz-title long-title nil)))
+ "the title itself is cut to the limit")
+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped))
+ "nothing was dropped -- there was no link to begin with")
+ (mt/with-log-messages-for-level [messages [metabase.slackbot.streaming :info]]
+ (#'slackbot.streaming/format-viz-title long-title nil)
+ (is (some (fn [{:keys [message]}]
+ (str/includes? message "no link to drop"))
+ (messages))
+ "and the log says so, rather than claiming a link was dropped")))))))
+
+(deftest viz-image-alt-text-fits-slack-limit-test
+ (testing "alt_text is capped to Slack's tighter image limit, which a title can exceed alone (BOT-1606)"
+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]
+ (mt/with-dynamic-fn-redefs [channel.slack/upload-file! (constantly {:id "F123" :url "https://x/y.png"})]
+ ;; 2500 chars clears the 3000 section limit untouched, so only the alt_text cap catches it.
+ (let [long-title (apply str (repeat 2500 "T"))
+ blocks (#'slackbot.streaming/viz-output->blocks
+ {:type :image :content (byte-array [1 2 3])}
+ "chart" long-title nil)
+ image (second blocks)]
+ (is (= ["section" "image"] (mapv :type blocks)))
+ (is (= 2500 (count (get-in (first blocks) [:text :text])))
+ "the section text is under its own limit, so nothing there would have caught this")
+ (is (= 2000 (count (:alt_text image)))
+ "alt_text is cut to Slack's 2000-character limit")
+ (is (str/starts-with? long-title (:alt_text image)))
+ (testing "a title that fits is left alone"
+ (let [ok (#'slackbot.streaming/viz-output->blocks
+ {:type :image :content (byte-array [1 2 3])}
+ "chart" "Revenue by month" nil)]
+ (is (= "Revenue by month" (:alt_text (second ok)))))))))))
(deftest feedback-blocks-test
(testing "feedback-blocks generates correct Slack context_actions block with feedback_buttons"
diff --git a/test/metabase/slackbot/test_util.clj b/test/metabase/slackbot/test_util.clj
index 89f5ec2e0f6..0f77effa041 100644
--- a/test/metabase/slackbot/test_util.clj
+++ b/test/metabase/slackbot/test_util.clj
@@ -3,9 +3,11 @@
(:require
[buddy.core.codecs :as codecs]
[buddy.core.mac :as mac]
+ [clojure.string :as str]
[metabase.channel.slack :as channel.slack]
[metabase.metabot.agent.core :as agent]
[metabase.slackbot.api :as slackbot]
+ [metabase.slackbot.channel :as slackbot.channel]
[metabase.slackbot.client :as slackbot.client]
[metabase.slackbot.config :as slackbot.config]
[metabase.slackbot.query :as slackbot.query]
@@ -45,6 +47,35 @@
:url_private "https://files.slack.com/files/data.csv"
:size 100})
+;; Aliased rather than restated, so the harness cannot drift from what the code enforces.
+(def slack-section-text-limit
+ "Alias for [[metabase.slackbot.channel/section-text-limit]]."
+ slackbot.channel/section-text-limit)
+
+(def oversized-answer
+ "An answer comfortably past [[slack-section-text-limit]].
+ Numbered line by line, so a truncated copy can be checked against the original."
+ (str/join "\n" (map #(format "Line %04d of a rather long answer." %) (range 200))))
+
+(defn oversized-section-error
+ "The `chat.postMessage` rejection Slack returns for an oversized `section` block in `blocks`.
+ Nil when every block is within [[slack-section-text-limit]]. Models that one rule -- the one
+ BOT-1606 is about -- not Block Kit validation at large."
+ [blocks]
+ ;; The mocked client accepts anything, so tests that care about this rule come through here.
+ (when-let [idx (first (keep-indexed (fn [idx block]
+ (when (and (= "section" (:type block))
+ (> (count (get-in block [:text :text] ""))
+ slack-section-text-limit))
+ idx))
+ blocks))]
+ {:ok false
+ :error "invalid_blocks"
+ :response_metadata
+ {:messages [(format "[ERROR] failed to match all allowed schemas [json-pointer:/blocks/%d/text]" idx)
+ (format "[ERROR] must be less than %d characters [json-pointer:/blocks/%d/text/text]"
+ (inc slack-section-text-limit) idx)]}}))
+
(defmacro with-ensure-encryption
"Use the existing encryption key if one is configured, otherwise set a test key.
Avoids conflicts with encrypted settings in the DB that were written with the real key."