The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1606: Truncate over-long Metabot answers instead of going silent (#80105)
Paolo Rascunà638d0e4cb28b
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex aaf9a97bcf3..6a9ceb38ad2 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -732,6 +732,15 @@(prometheus/counter :metabase-slackbot/file-uploads{:description "Number of file uploads via the Slack bot.":labels [:result]})+ (prometheus/counter :metabase-slackbot/responses-truncated+ {:description (str "Number of Slack bot responses truncated because they exceeded "+ "Slack's message limits.")})+ (prometheus/counter :metabase-slackbot/viz-links-dropped+ {:description (str "Number of visualization query links dropped from Slack bot messages "+ "because the linked title exceeded Slack's block limits.")})+ (prometheus/counter :metabase-slackbot/responses-undeliverable+ {:description (str "Number of Slack bot responses that reached the user as nothing at all, "+ "because the plain-text fallback failed too.")});; metabot / LLM agent metrics(prometheus/counter :metabase-metabot/llm-requests{:description "LLM provider API requests"diff --git a/src/metabase/slackbot/channel.clj b/src/metabase/slackbot/channel.cljindex 0e3dd28bd00..8b4556b846f 100644--- a/src/metabase/slackbot/channel.clj+++ b/src/metabase/slackbot/channel.clj@@ -2,9 +2,12 @@"Visible Slack channel reply/update flow for metabot."(:require[clojure.string :as str]+ [metabase.analytics-interface.core :as analytics][metabase.metabot.persistence :as metabot.persistence][metabase.slackbot.client :as slackbot.client]- [metabase.util.log :as log]))+ [metabase.system.core :as system]+ [metabase.util.log :as log]+ [metabase.util.string :as u.str]))(set! *warn-on-reflection* true)@@ -20,6 +23,12 @@[prompt](str prompt channel-response-style-suffix))+(def section-text-limit+ "Slack rejects a `section` block whose `text.text` exceeds this many characters.+ The whole `chat.postMessage` fails with `invalid_blocks`.+ See https://docs.slack.dev/reference/block-kit/blocks/section-block."+ 3000)+(defn- final-text-blocks"Build the leading text block(s) for a finalized non-streaming Slack message."[text]@@ -29,6 +38,30 @@:text {:type "mrkdwn":text text}}]))+(defn- truncation-notice+ "Copy explaining that an answer was cut short.+ `url` points at the Metabase instance; nil leaves the sentence standing without a link."+ [url]+ (str "_This answer was longer than " section-text-limit " characters. That is too long to post in "+ "Slack, so I cut it short._\n\n"+ "Ask a narrower question so the answer comes back smaller"+ (if url+ (str ", or head to <" url "|Metabase> and try again.")+ ".")))++(defn- truncation-block+ "A muted aside explaining a cut answer, for the same message the answer was posted in."+ []+ ;; `context` rather than a second section: Slack renders it small and grey, so it reads as an+ ;; aside about the answer rather than as part of it.+ ;;+ ;; The instance home page, not the conversation that produced the answer: picking a Slack thread+ ;; back up on the web is not supported, so a deep link would only lead somewhere that cannot help.+ ;; `site-url` is guaranteed to carry no trailing slash, so it stands as a link on its own.+ {:type "context"+ :elements [{:type "mrkdwn"+ :text (truncation-notice (system/site-url))}]})+(defn- make-channel-callbacks"Create callback functions for channel replies.Uses the Slack assistant setStatus API for progress indication.@@ -87,13 +120,26 @@(set-status! "Rendering results..."))(let [{:keys [blocks errors]} (collect-viz-blocks @prefetched-viz)answer-text (str/trim @current-text)- final-text (if (or (seq answer-text) (seq blocks))+ answer (if (or (seq answer-text) (seq blocks))answer-text"I wasn't able to generate a response. Please try again.")- final-blocks (into (into (final-text-blocks final-text) blocks)- (feedback-blocks conversation-id message-external-id))+ ;; `final-text` doubles as the message's `:text` -- Slack's notification preview, and+ ;; what `streaming/thread->history` replays to the model as the assistant's own words.+ ;; The notice stays out of it, so the model is never told it said this.+ final-text (u.str/elide answer section-text-limit)+ ;; Derived rather than re-tested: `elide` returns `answer` itself when it fits, so+ ;; the notice cannot disagree with whether a cut actually happened.+ truncated? (not= final-text answer)+ final-blocks (-> (final-text-blocks final-text)+ (cond-> truncated? (conj (truncation-block)))+ (into blocks)+ (into (feedback-blocks conversation-id message-external-id)))res (slackbot.client/post-thread-reply client {:channel channel :thread_ts thread-ts}final-text :blocks final-blocks)]+ (when truncated?+ (log/infof "[slackbot] channel answer truncated (answer_length=%d limit=%d)"+ (count answer) section-text-limit)+ (analytics/inc! :metabase-slackbot/responses-truncated))(when-let [res-ts (:ts res)](metabot.persistence/set-response-slack-msg-id! assistant-msg-id res-ts))(when-not (:ok res)@@ -101,7 +147,16 @@(:error res)(count final-blocks)(pr-str (mapv :type final-blocks))- (pr-str (get-in res [:response_metadata :messages]))))+ (pr-str (get-in res [:response_metadata :messages])))+ ;; The blocks were rejected for a reason we did not anticipate. A plain-text reply+ ;; carries no blocks for Slack to reject, so the user gets something back rather than+ ;; silence (BOT-1606).+ (let [fallback (slackbot.client/post-thread-reply+ client message-ctx+ "I generated a response, but Slack could not render it. Please try again.")]+ (when-not (:ok fallback)+ (log/errorf "[slackbot] channel fallback post-message failed: %s" (:error fallback))+ (analytics/inc! :metabase-slackbot/responses-undeliverable))))(doseq [e errors](post-viz-error! client channel thread-ts e))))(catch Exception ediff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.cljindex 97a177b41bf..34deb376915 100644--- a/src/metabase/slackbot/streaming.clj+++ b/src/metabase/slackbot/streaming.clj@@ -4,6 +4,7 @@(:require[clojure.string :as str][java-time.api :as t]+ [metabase.analytics-interface.core :as analytics][metabase.api.common :as api][metabase.channel.slack :as channel.slack][metabase.llm.provider :as llm.provider]@@ -24,7 +25,8 @@[metabase.system.core :as system][metabase.util :as u][metabase.util.json :as json]- [metabase.util.log :as log])+ [metabase.util.log :as log]+ [metabase.util.string :as u.str])(:import(java.util.concurrentCallable@@ -113,16 +115,41 @@(str/replace ">" ">")(str/replace "|" "\u2502")))+(def ^:private image-alt-text-limit+ "Slack rejects an `image` block whose `alt_text` exceeds this many characters.+ Tighter than [[slackbot.channel/section-text-limit]], so a title can fit the section and still+ fail here.+ See https://docs.slack.dev/reference/block-kit/blocks/image-block."+ 2000)+(defn- format-viz-title- "Build the title text for a visualization message.- Combines the title with a link to the query in Metabase."+ "Build the title text for a visualization message, combining the title with a link to the query.+ Returns nil when nothing is left to show, and never exceeds+ [[slackbot.channel/section-text-limit]]."[title link]- (let [full-link (when link (str (system/site-url) link))]- (cond- (and title full-link) (str "\ud83d\udcca <" full-link "|" (escape-slack-link-text title) ">")- title title- full-link (str "\ud83d\udcca <" full-link "|Open in Metabase>")- :else nil)))+ (let [limit slackbot.channel/section-text-limit+ full-link (when link (str (system/site-url) link))+ text (cond+ (and title full-link) (str "\ud83d\udcca <" full-link "|" (escape-slack-link-text title) ">")+ title title+ full-link (str "\ud83d\udcca <" full-link "|Open in Metabase>")+ :else nil)]+ ;; An adhoc `/question#<base64-query>` link can run past the section limit on its own, and Slack+ ;; then rejects the whole message (BOT-1606). A cut URL is a broken URL, so drop the link rather+ ;; than truncate it; with no title left, `viz-output->blocks` falls back to "Query results".+ (if (and text (> (count text) limit))+ (do+ ;; Either way the answer is the same elided title -- only what we report differs, because+ ;; with no link there is nothing to drop and the counter would overstate what happened.+ (if full-link+ (do+ (log/infof "[slackbot] viz title over limit, dropping query link (text_length=%d link_length=%d limit=%d)"+ (count text) (count full-link) limit)+ (analytics/inc! :metabase-slackbot/viz-links-dropped))+ (log/infof "[slackbot] viz title over limit with no link to drop, eliding it (title_length=%d limit=%d)"+ (count text) limit))+ (u.str/elide title limit))+ text)))(defn- viz-output->blocks"Build blocks for a visualization to be included in the finalized stop-stream message."@@ -138,7 +165,10 @@:text {:type "mrkdwn" :text text}}{:type "image":slack_file {:id id}- :alt_text (or title filename "Visualization")}]]+ ;; Plain text, so it is cut rather than elided -- no ellipsis to add.+ :alt_text (or (some-> title (u.str/limit-chars image-alt-text-limit))+ filename+ "Visualization")}]]blocks))))(def ^:private tool-friendly-names@@ -604,7 +634,9 @@message-ctx"I generated a response, but Slack could not render it. Please try again.")](when-not (:ok fallback-result)- (log/errorf "[slackbot] fallback post-message failed after stop-stream error: %s" (:error fallback-result)))))+ (log/errorf "[slackbot] fallback post-message failed after stop-stream error: %s"+ (:error fallback-result))+ (analytics/inc! :metabase-slackbot/responses-undeliverable))))(doseq [e errors](post-viz-error! client channel thread-ts e)))(slackbot.client/post-thread-reply client message-ctx "I wasn't able to generate a response. Please try again.")))diff --git a/test/metabase/slackbot/api_test.clj b/test/metabase/slackbot/api_test.cljindex cf756944f96..33bfbff4cab 100644--- a/test/metabase/slackbot/api_test.clj+++ b/test/metabase/slackbot/api_test.clj@@ -206,6 +206,39 @@(let [msg (t2/select-one :model/MetabotMessage :channel_id channel-id :role "assistant")](is (some? (:slack_msg_id msg))))))))))))+(deftest app-mention-long-answer-fits-slack-blocks-test+ (testing "POST /events with app_mention truncates a long answer so Slack accepts it (BOT-1606)"+ (tu/with-slackbot-setup+ (let [channel-id "C-LONG-ANSWER-TEST"+ event-body (assoc-in tu/base-mention-event [:event :channel] channel-id)]+ (tu/with-slackbot-mocks+ {:ai-text tu/oversized-answer}+ (fn [{:keys [post-calls]}]+ (let [response (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body)]+ (is (= "ok" response))+ (u/poll {:thunk #(t2/select-one :model/MetabotMessage+ :channel_id channel-id :role "assistant"+ :slack_msg_id [:not= nil])+ :done? some?+ :timeout-ms 5000})+ (is (= 1 (count @post-calls)))+ (let [blocks (:blocks (first @post-calls))]+ (testing "no section block is over the limit -- unlike the untruncated answer"+ (is (nil? (tu/oversized-section-error blocks)))+ (is (some? (tu/oversized-section-error [{:type "section"+ :text {:type "mrkdwn" :text tu/oversized-answer}}]))+ "the answer really is past the limit, so the case under test is the real one"))+ (testing "the answer is cut to the limit, and the message says why"+ (is (= tu/slack-section-text-limit+ (count (get-in (first blocks) [:text :text]))))+ (is (some (fn [block]+ (and (= "context" (:type block))+ (str/includes? (get-in block [:elements 0 :text])+ "too long to post in Slack")))+ blocks)))))))))))+(deftest stream-start-failure-test(testing "When start-stream fails, falls back to a regular message"(tu/with-slackbot-setup@@ -229,6 +262,29 @@(testing "stop-stream is never called"(is (= 0 (count @stop-stream-calls))))))))))))+;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.+(deftest dm-response-undeliverable-metric-test+ (testing "a DM whose stream and plain-text fallback both fail is counted as undeliverable"+ (tu/with-slackbot-setup+ (mt/with-prometheus-system! [_ system]+ (let [event-body tu/base-dm-event]+ (tu/with-slackbot-mocks+ {:ai-text "Here is your answer"}+ (fn [_]+ ;; `post-thread-reply` delegates to `post-message`, so failing that fails the+ ;; fallback too -- the user ends up with nothing at all.+ (mt/with-dynamic-fn-redefs+ [slackbot.client/stop-stream (constantly {:ok false :error "invalid_blocks"})+ slackbot.client/post-message (constantly {:ok false :error "channel_not_found"})]+ (let [response (mt/client :post 200 "metabot/slack/events"+ (tu/slack-request-options event-body)+ event-body)]+ (is (= "ok" response))+ (u/poll {:thunk #(mt/metric-value system :metabase-slackbot/responses-undeliverable)+ :done? pos?+ :timeout-ms 5000})+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))))))))))+(deftest ai-request-error-stops-stream-test(testing "When the agent loop throws after the stream has started, the stream is stopped"(tu/with-slackbot-setupdiff --git a/test/metabase/slackbot/channel_test.clj b/test/metabase/slackbot/channel_test.cljindex 8b1fe6db696..7736829ff00 100644--- a/test/metabase/slackbot/channel_test.clj+++ b/test/metabase/slackbot/channel_test.clj@@ -1,9 +1,12 @@(ns metabase.slackbot.channel-test(:require+ [clojure.string :as str][clojure.test :refer :all]+ [metabase.analytics.prometheus :as prometheus][metabase.metabot.persistence :as metabot.persistence][metabase.slackbot.channel :as slackbot.channel][metabase.slackbot.client :as slackbot.client]+ [metabase.slackbot.test-util :as slackbot.tu][metabase.test :as mt]))(deftest channel-response-passes-slack-metadata-for-deep-linking-test@@ -43,3 +46,182 @@(select-keys @request-opts [:team-id :thread-ts :req-slack-msg-id])))(is (= {:msg-id 42 :slack-msg-id "1700000000.000002"}@backfill-args))))++;;; ---------------------------------- BOT-1606: over-long answers ----------------------------------++(def ^:private conversation-id "0f8a1c3e-0000-4000-8000-000000000001")++(def ^:private test-site-url "https://metabase.example.com")++(def ^:private viz-blocks+ [{:type "section" :text {:type "mrkdwn" :text "*Orders by month*"}}+ {:type "table" :rows []}])++(def ^:private feedback-blocks+ [{:type "context_actions" :elements []}])++(defn- drive-channel-response!+ "Drive [[slackbot.channel/send-channel-response]] to completion with `answer` as the model's text.+ `reply-result` stands in for what Slack returns from `post-thread-reply`, and is called with the+ number of posts made so far. Returns `{:posts [{:text .. :blocks ..} ..] :backfill ..}`, in the+ order the posts were made."+ [answer reply-result]+ (let [posts (atom [])+ backfill (atom nil)]+ (mt/with-dynamic-fn-redefs [slackbot.client/set-status (constantly {:ok true})+ slackbot.client/post-thread-reply+ (fn [_client _message-ctx text & {:keys [blocks]}]+ (reply-result (count (swap! posts conj {:text text :blocks blocks}))))+ metabot.persistence/set-response-slack-msg-id!+ (fn [msg-id slack-msg-id]+ (reset! backfill {:msg-id msg-id :slack-msg-id slack-msg-id}))]+ (slackbot.channel/send-channel-response+ {}+ {:ts "1700000000.000001"}+ nil+ {:channel-id "C123"+ :message-ctx {:channel "C123" :thread_ts "1700000000.000001"}+ :channel "C123"+ :thread-ts "1700000000.000001"+ :auth-info {:team_id "T123"}+ :thread {:messages []}+ :bot-user-id "U999"+ :prompt "hello"+ :conversation-id conversation-id}+ {:tool-name->friendly {}+ :make-streaming-ai-request (fn [& args]+ ((:on-text (last args)) answer)+ {:msg-id 42+ :external-id "message-external-id"})+ :collect-viz-blocks (constantly {:blocks viz-blocks :errors []})+ :feedback-blocks (constantly feedback-blocks)+ :post-viz-error! (constantly nil)+ :make-viz-prefetch-callback (constantly (fn [& _]))+ :cancel-prefetched-viz! (constantly nil)}))+ {:posts @posts :backfill @backfill}))++(defn- send-channel-response!+ "Drive [[slackbot.channel/send-channel-response]] with a Slack that accepts everything.+ Returns `{:text .. :blocks ..}` as posted, plus the `:backfill` it recorded."+ [answer]+ (let [{:keys [posts backfill]} (drive-channel-response!+ answer+ (constantly {:ok true :ts "1700000000.000002"}))]+ (assoc (last posts) :backfill backfill)))++(deftest ^:parallel truncation-notice-test+ (testing "the notice points at the instance, and says what to do about the cut"+ (is (= (str "_This answer was longer than 3000 characters. That is too long to post in Slack, "+ "so I cut it short._\n\n"+ "Ask a narrower question so the answer comes back smaller"+ ", or head to <https://metabase.example.com|Metabase> and try again.")+ (#'slackbot.channel/truncation-notice test-site-url))))+ (testing "an instance with no site URL still gets a sentence, just without the link"+ (is (= (str "_This answer was longer than 3000 characters. That is too long to post in Slack, "+ "so I cut it short._\n\n"+ "Ask a narrower question so the answer comes back smaller.")+ (#'slackbot.channel/truncation-notice nil))))+ (testing "the notice never deep-links to the conversation -- continuing it on the web is unsupported"+ (is (not (str/includes? (#'slackbot.channel/truncation-notice test-site-url)+ "/metabot/conversation/")))))++(deftest channel-response-truncates-oversized-answer-test+ (testing "an oversized answer is cut, and the same message explains why (BOT-1606)"+ (mt/with-temporary-setting-values [site-url test-site-url]+ (let [{:keys [text blocks backfill]} (send-channel-response! slackbot.tu/oversized-answer)+ [answer-block notice-block] blocks+ section-text (get-in answer-block [:text :text])]+ (testing "no section block is over the limit -- so Slack no longer rejects the message"+ (is (nil? (slackbot.tu/oversized-section-error blocks))))+ (testing "the answer rides one mrkdwn section block, cut to the limit"+ (is (= "section" (:type answer-block)))+ (is (= "mrkdwn" (get-in answer-block [:text :type])))+ (is (= slackbot.tu/slack-section-text-limit (count section-text)))+ (is (str/ends-with? section-text "..."))+ (is (str/starts-with? slackbot.tu/oversized-answer (str/replace section-text #"\.\.\.$" ""))+ "what survives the cut is a genuine prefix of the answer"))+ (testing "the notice rides in a context block of the same message, linking to the instance"+ (is (= "context" (:type notice-block)))+ (is (= [{:type "mrkdwn"+ :text (#'slackbot.channel/truncation-notice test-site-url)}]+ (:elements notice-block)))+ (is (not (str/includes? (get-in notice-block [:elements 0 :text]) "/metabot/conversation/"))+ "the link is the home page, not a conversation we cannot continue on the web"))+ (testing "the visualizations and the feedback buttons still ride the message"+ (is (= ["section" "context" "section" "table" "context_actions"]+ (mapv :type blocks))))+ (testing "the notice stays out of `:text`, which `thread->history` replays back to the model"+ (is (= section-text text))+ (is (not (str/includes? text "too long to post in Slack"))))+ (testing "one message is posted, and its ts is persisted -- it carries the feedback buttons"+ (is (= {:msg-id 42 :slack-msg-id "1700000000.000002"} backfill)))))))++(deftest channel-response-truncation-notice-without-site-url-test+ (testing "an instance with no site URL still gets the notice, just without a link"+ (mt/with-temporary-setting-values [site-url nil]+ (let [{:keys [blocks]} (send-channel-response! slackbot.tu/oversized-answer)+ notice-block (second blocks)+ notice (get-in notice-block [:elements 0 :text])]+ (is (= "context" (:type notice-block)))+ (is (str/includes? notice "too long to post in Slack"))+ (is (not (str/includes? notice "<"))+ "nothing to link to, so the sentence stands on its own")))))++(deftest channel-response-does-not-truncate-when-it-fits-test+ (testing "an answer within the limit is posted whole, with no notice"+ (let [answer "Orders peaked in March. *1,204* of them."+ {:keys [text blocks]} (send-channel-response! answer)]+ (is (nil? (slackbot.tu/oversized-section-error blocks)))+ (is (= answer (get-in (first blocks) [:text :text])))+ (is (= answer text))+ (is (= ["section" "section" "table" "context_actions"] (mapv :type blocks))+ "no context block, because nothing was cut"))))++;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.+(deftest channel-response-truncation-metric-test+ (mt/with-prometheus-system! [_ system]+ (mt/with-temporary-setting-values [site-url test-site-url]+ (testing "truncating an answer increments the truncation counter"+ (send-channel-response! slackbot.tu/oversized-answer)+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-truncated))))+ (testing "an answer that fits does not"+ (prometheus/clear! :metabase-slackbot/responses-truncated)+ (send-channel-response! "Short answer.")+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-truncated)))))))++(deftest channel-response-falls-back-to-plain-text-when-blocks-are-rejected-test+ (testing "a message Slack rejects still reaches the user as plain text, rather than silence"+ (let [{:keys [posts]} (drive-channel-response!+ "Orders peaked in March."+ (fn [n] (if (= n 1)+ {:ok false :error "invalid_blocks"}+ {:ok true :ts "1700000000.000002"})))+ [rejected fallback] posts]+ (is (= 2 (count posts))+ "the rejected post is followed by a plain-text retry")+ (is (seq (:blocks rejected))+ "the first attempt carried blocks")+ (is (nil? (:blocks fallback))+ "the fallback carries none, so there is nothing left for Slack to reject")+ (is (= "I generated a response, but Slack could not render it. Please try again."+ (:text fallback)))))+ (testing "a message Slack accepts is posted once, with no fallback"+ (let [{:keys [posts]} (drive-channel-response!+ "Orders peaked in March."+ (constantly {:ok true :ts "1700000000.000002"}))]+ (is (= 1 (count posts))))))++;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.+(deftest channel-response-undeliverable-metric-test+ (mt/with-prometheus-system! [_ system]+ (testing "a response that fails even as plain text is counted as undeliverable"+ (drive-channel-response! "Orders peaked in March."+ (constantly {:ok false :error "channel_not_found"}))+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))+ (testing "a response saved by the fallback is not -- the user did get a reply"+ (prometheus/clear! :metabase-slackbot/responses-undeliverable)+ (drive-channel-response! "Orders peaked in March."+ (fn [n] (if (= n 1)+ {:ok false :error "invalid_blocks"}+ {:ok true :ts "1700000000.000002"})))+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-undeliverable))))))diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.cljindex fe4abdd1ff0..766caa02111 100644--- a/test/metabase/slackbot/streaming_test.clj+++ b/test/metabase/slackbot/streaming_test.clj@@ -1,6 +1,9 @@(ns metabase.slackbot.streaming-test(:require+ [clojure.string :as str][clojure.test :refer :all]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.channel.slack :as channel.slack][metabase.metabot.persistence :as metabot.persistence][metabase.metabot.settings :as metabot.settings][metabase.premium-features.core :as premium-features]@@ -107,7 +110,97 @@(#'slackbot.streaming/format-viz-title "Foo <Bar> | Baz" "/question/42"))))(testing "title-only does not escape (no link syntax)"(is (= "Sales & Revenue"- (#'slackbot.streaming/format-viz-title "Sales & Revenue" nil)))))))+ (#'slackbot.streaming/format-viz-title "Sales & Revenue" nil))))+ (testing "an over-long link is dropped, not cut -- a truncated URL is a broken URL (BOT-1606)"+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))]+ (is (= "My Chart"+ (#'slackbot.streaming/format-viz-title "My Chart" huge-link))+ "the title survives; the link that cannot fit does not")+ (is (nil? (#'slackbot.streaming/format-viz-title nil huge-link))+ "nothing left to keep, so viz-output->blocks falls back to Query results"))))))++(deftest viz-blocks-fit-slack-section-limit-test+ (testing "a viz whose query link runs past the section limit still posts (BOT-1606)"+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))+ blocks (#'slackbot.streaming/viz-output->blocks+ {:type :table :content [{:type "table" :rows []}]}+ "revenue-by-month"+ "Revenue by month"+ huge-link)]+ (is (nil? (tu/oversized-section-error blocks))+ "Slack no longer rejects the whole message")+ (is (some? (tu/oversized-section-error+ [{:type "section"+ :text {:type "mrkdwn"+ :text (str "📊 <https://metabase.example.com" huge-link "|Revenue by month>")}}]))+ "the linked title really is past the limit, so the case under test is the real one")+ (is (= "Revenue by month" (get-in (first blocks) [:text :text])))+ (is (= ["section" "table"] (mapv :type blocks)))))))++;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.+(deftest viz-title-over-limit-is-observable-test+ (testing "dropping an over-long query link is logged and counted (BOT-1606)"+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]+ (let [huge-link (str "/question#" (apply str (repeat 4000 "Q")))]+ (testing "the drop is recorded in the log"+ ;; `log/infof` only evaluates its args once the level is enabled, so without this the+ ;; line never runs under test and a bad format arg would ship unnoticed.+ (mt/with-log-messages-for-level [messages [metabase.slackbot.streaming :info]]+ (#'slackbot.streaming/format-viz-title "Revenue by month" huge-link)+ (is (some (fn [{:keys [message]}]+ (str/includes? message "viz title over limit, dropping query link"))+ (messages)))))+ (testing "and counted on its own metric, separate from answer truncation"+ (mt/with-prometheus-system! [_ system]+ (#'slackbot.streaming/format-viz-title "Revenue by month" huge-link)+ (is (= 1.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped)))+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/responses-truncated))+ "an over-long link is not an over-long answer")+ (testing "a link that fits is not counted"+ (prometheus/clear! :metabase-slackbot/viz-links-dropped)+ (#'slackbot.streaming/format-viz-title "Revenue by month" "/question/42")+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped))))))))))++;; Not ^:parallel: `with-prometheus-system!` redefs a process-global var.+(deftest viz-title-over-limit-without-link-is-not-a-dropped-link-test+ (testing "an over-long title with no link to drop is elided, but not counted as a dropped link"+ (mt/with-prometheus-system! [_ system]+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]+ (let [long-title (apply str (repeat 4000 "T"))]+ (is (= tu/slack-section-text-limit+ (count (#'slackbot.streaming/format-viz-title long-title nil)))+ "the title itself is cut to the limit")+ (is (= 0.0 (mt/metric-value system :metabase-slackbot/viz-links-dropped))+ "nothing was dropped -- there was no link to begin with")+ (mt/with-log-messages-for-level [messages [metabase.slackbot.streaming :info]]+ (#'slackbot.streaming/format-viz-title long-title nil)+ (is (some (fn [{:keys [message]}]+ (str/includes? message "no link to drop"))+ (messages))+ "and the log says so, rather than claiming a link was dropped")))))))++(deftest viz-image-alt-text-fits-slack-limit-test+ (testing "alt_text is capped to Slack's tighter image limit, which a title can exceed alone (BOT-1606)"+ (mt/with-temporary-setting-values [site-url "https://metabase.example.com"]+ (mt/with-dynamic-fn-redefs [channel.slack/upload-file! (constantly {:id "F123" :url "https://x/y.png"})]+ ;; 2500 chars clears the 3000 section limit untouched, so only the alt_text cap catches it.+ (let [long-title (apply str (repeat 2500 "T"))+ blocks (#'slackbot.streaming/viz-output->blocks+ {:type :image :content (byte-array [1 2 3])}+ "chart" long-title nil)+ image (second blocks)]+ (is (= ["section" "image"] (mapv :type blocks)))+ (is (= 2500 (count (get-in (first blocks) [:text :text])))+ "the section text is under its own limit, so nothing there would have caught this")+ (is (= 2000 (count (:alt_text image)))+ "alt_text is cut to Slack's 2000-character limit")+ (is (str/starts-with? long-title (:alt_text image)))+ (testing "a title that fits is left alone"+ (let [ok (#'slackbot.streaming/viz-output->blocks+ {:type :image :content (byte-array [1 2 3])}+ "chart" "Revenue by month" nil)]+ (is (= "Revenue by month" (:alt_text (second ok)))))))))))(deftest feedback-blocks-test(testing "feedback-blocks generates correct Slack context_actions block with feedback_buttons"diff --git a/test/metabase/slackbot/test_util.clj b/test/metabase/slackbot/test_util.cljindex 89f5ec2e0f6..0f77effa041 100644--- a/test/metabase/slackbot/test_util.clj+++ b/test/metabase/slackbot/test_util.clj@@ -3,9 +3,11 @@(:require[buddy.core.codecs :as codecs][buddy.core.mac :as mac]+ [clojure.string :as str][metabase.channel.slack :as channel.slack][metabase.metabot.agent.core :as agent][metabase.slackbot.api :as slackbot]+ [metabase.slackbot.channel :as slackbot.channel][metabase.slackbot.client :as slackbot.client][metabase.slackbot.config :as slackbot.config][metabase.slackbot.query :as slackbot.query]@@ -45,6 +47,35 @@:url_private "https://files.slack.com/files/data.csv":size 100})+;; Aliased rather than restated, so the harness cannot drift from what the code enforces.+(def slack-section-text-limit+ "Alias for [[metabase.slackbot.channel/section-text-limit]]."+ slackbot.channel/section-text-limit)++(def oversized-answer+ "An answer comfortably past [[slack-section-text-limit]].+ Numbered line by line, so a truncated copy can be checked against the original."+ (str/join "\n" (map #(format "Line %04d of a rather long answer." %) (range 200))))++(defn oversized-section-error+ "The `chat.postMessage` rejection Slack returns for an oversized `section` block in `blocks`.+ Nil when every block is within [[slack-section-text-limit]]. Models that one rule -- the one+ BOT-1606 is about -- not Block Kit validation at large."+ [blocks]+ ;; The mocked client accepts anything, so tests that care about this rule come through here.+ (when-let [idx (first (keep-indexed (fn [idx block]+ (when (and (= "section" (:type block))+ (> (count (get-in block [:text :text] ""))+ slack-section-text-limit))+ idx))+ blocks))]+ {:ok false+ :error "invalid_blocks"+ :response_metadata+ {:messages [(format "[ERROR] failed to match all allowed schemas [json-pointer:/blocks/%d/text]" idx)+ (format "[ERROR] must be less than %d characters [json-pointer:/blocks/%d/text/text]"+ (inc slack-section-text-limit) idx)]}}))+(defmacro with-ensure-encryption"Use the existing encryption key if one is configured, otherwise set a test key.Avoids conflicts with encrypted settings in the DB that were written with the real key."