Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1573: Slackbot posts specific copy for known error codes (#81200)

Andrei Simionescuf4b04b69669b
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index 021b2e88d00..2f15996ee43 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -454,14 +454,14 @@
(defn- check-metabot-access!
"Throw a 403 if the user's metabot permissions do not grant access to the
- requested profile. The base + profile-specific gating policy lives in
- [[scope/missing-permission]], shared with [[metabase.metabot.self]]."
+ requested profile."
[profile-id perms]
(when-let [missing (scope/missing-permission perms (profile-id->required-permission profile-id))]
- (api/check false
- [403 (if (= missing :permission/metabot)
+ (throw (ex-info (if (= missing :permission/metabot)
"You do not have permission to use the AI assistant."
- (format "You do not have permission to use the %s assistant." (name profile-id)))])))
+ (format "You do not have permission to use the %s assistant." (name profile-id)))
+ {:status-code 403
+ :type :metabot/permission-denied}))))
(defn- init-agent
"Initialize agent state."
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index a6c26a6b26d..4ab5c2b2e1b 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -118,6 +118,7 @@
(pr-str (llm.provider/model-ref->connection-key s)))
{:status-code 400
:api-error true
+ :error-code :llm-not-configured
:model-ref s})))]
{:provider type
:stream-fn (resolve-adapter type)
diff --git a/src/metabase/slackbot/channel.clj b/src/metabase/slackbot/channel.clj
index 20140218ccd..929b2b261a4 100644
--- a/src/metabase/slackbot/channel.clj
+++ b/src/metabase/slackbot/channel.clj
@@ -100,7 +100,7 @@
[client event extra-history {:keys [channel-id message-ctx channel thread-ts auth-info thread bot-user-id prompt conversation-id]}
{:keys [tool-name->friendly
make-streaming-ai-request collect-viz-blocks feedback-blocks post-viz-error!
- make-viz-prefetch-callback cancel-prefetched-viz!]}]
+ make-viz-prefetch-callback cancel-prefetched-viz! error-message]}]
(let [{:keys [on-text on-tool-start set-status! current-text]}
(make-channel-callbacks client {:channel channel
:thread-ts thread-ts
@@ -173,7 +173,6 @@
(cancel-prefetched-viz! prefetched-viz)
(log/errorf "[slackbot] Error in channel response: %s" (ex-message e))
(set-status! nil)
- (let [res (slackbot.client/post-thread-reply client message-ctx
- "Something went wrong. Please try again.")]
+ (let [res (slackbot.client/post-thread-reply client message-ctx (error-message (ex-data e)))]
(when-not (:ok res)
(log/errorf "[slackbot] channel error post-message failed: %s" (:error res))))))))
diff --git a/src/metabase/slackbot/streaming.clj b/src/metabase/slackbot/streaming.clj
index 0f561e94ce9..c74dca93eb2 100644
--- a/src/metabase/slackbot/streaming.clj
+++ b/src/metabase/slackbot/streaming.clj
@@ -189,6 +189,49 @@
headroom for tool-update and other non-text API calls."
600)
+(def ^:private generic-error-message
+ "Something went wrong. Please try again.")
+
+(def ^:private provider-config-error-codes
+ "Error codes that mean the LLM provider connection is misconfigured.
+
+ Thrown by [[metabase.metabot.self/parse-provider-model]] and by the provider
+ adapters' own setup validation; all deserve the same check-your-AI-settings copy."
+ #{"llm-not-configured" "api-key-missing" "credentials-unavailable" "base-url-missing"
+ "model-missing" "proxy-unsupported" "proxy-not-configured" "invalid-service-account-key"
+ "not-a-service-account-key" "invalid-location" "project-id-required"
+ "invalid-project-id" "invalid-model" "unsupported-model" "invalid-region"})
+
+(defn- known-error-message
+ "User-facing copy for a failure this namespace recognizes, or nil.
+
+ Failures are recognized by error code, and permission denials also by the
+ `:type :metabot/permission-denied` tag. `error` is a streamed `:error` part's
+ payload or a thrown exception's ex-data; errors the agent loop caught nest their
+ ex-data under `:data`. Only whitelisted markers get copy, and only the usage-limit
+ codes pass their server-authored message through: everything else, raw provider
+ errors and permission keywords included, must stay out of shared Slack channels."
+ [error]
+ (let [code (some-> (or (:error-code error) (get-in error [:data :error-code])) name)]
+ (cond
+ (or (= code "permission_denied")
+ (= :metabot/permission-denied (:type error))
+ (= :metabot/permission-denied (get-in error [:data :type])))
+ "You do not have permission to use the AI assistant."
+
+ (#{"metabase_ai_managed_locked" "ai_usage_limit_reached"} code)
+ (:message error)
+
+ (provider-config-error-codes code)
+ "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."
+
+ :else nil)))
+
+(defn- error-message
+ "[[known-error-message]] with the generic fallback applied."
+ [error]
+ (or (known-error-message error) generic-error-message))
+
(defn- make-streaming-ai-request
"Run the agent loop for a slackbot conversation, dispatching parts to callbacks.
@@ -277,7 +320,7 @@
:error
(when on-text
- (on-text "Something went wrong. Please try again."))
+ (on-text (error-message (:error part))))
nil)
nil)))]
@@ -573,12 +616,6 @@
:message_external_id message-external-id
:positive false})}}]}])
-(defn- free-limit-error-message
- [e]
- (let [{:keys [error-code message]} (ex-data e)]
- (when (= error-code "metabase_ai_managed_locked")
- (or message (ex-message e)))))
-
(defn- prepare-response-context
"Fetch thread/auth context shared by DM and channel delivery paths."
[client event]
@@ -660,21 +697,19 @@
(log/errorf "[slackbot] Error in streaming response: %s" (ex-message e))
(when-not (await-for slack-writer-await-timeout-ms slack-writer)
(log/warn "[slackbot] Timed out waiting for slack-writer agent to flush"))
- (if-let [{:keys [stream_ts channel]} @stream-state]
- (try
- (slackbot.client/append-markdown-text client channel stream_ts
- "\nSomething went wrong. Please try again.")
- (let [stop-result (slackbot.client/stop-stream client channel stream_ts)]
- (when-not (:ok stop-result)
- (log/warnf "[slackbot] stop-stream during error cleanup failed: %s" (:error stop-result))
- (let [fallback-result (slackbot.client/post-thread-reply client
- message-ctx
- "Something went wrong. Please try again.")]
- (when-not (:ok fallback-result)
- (log/errorf "[slackbot] cleanup fallback post-message failed: %s" (:error fallback-result))))))
- (catch Exception stop-e
- (log/debugf "[slackbot] Failed to stop stream during error cleanup: %s" (ex-message stop-e))))
- (slackbot.client/post-thread-reply client message-ctx "Something went wrong. Please try again."))))))
+ (let [message (error-message (ex-data e))]
+ (if-let [{:keys [stream_ts channel]} @stream-state]
+ (try
+ (slackbot.client/append-markdown-text client channel stream_ts (str "\n" message))
+ (let [stop-result (slackbot.client/stop-stream client channel stream_ts)]
+ (when-not (:ok stop-result)
+ (log/warnf "[slackbot] stop-stream during error cleanup failed: %s" (:error stop-result))
+ (let [fallback-result (slackbot.client/post-thread-reply client message-ctx message)]
+ (when-not (:ok fallback-result)
+ (log/errorf "[slackbot] cleanup fallback post-message failed: %s" (:error fallback-result))))))
+ (catch Exception stop-e
+ (log/debugf "[slackbot] Failed to stop stream during error cleanup: %s" (ex-message stop-e))))
+ (slackbot.client/post-thread-reply client message-ctx message)))))))
(defn send-response
"Send a metabot response using Slack delivery suited to the conversation type.
@@ -698,11 +733,12 @@
:feedback-blocks feedback-blocks
:post-viz-error! post-viz-error!
:make-viz-prefetch-callback make-viz-prefetch-callback
- :cancel-prefetched-viz! cancel-prefetched-viz!})))
+ :cancel-prefetched-viz! cancel-prefetched-viz!
+ :error-message error-message})))
(catch Exception e
- (if-let [message (free-limit-error-message e)]
+ (if-let [message (known-error-message (ex-data e))]
(let [result (slackbot.client/post-thread-reply client message-ctx message)]
(when-not (:ok result)
- (log/errorf "[slackbot] Failed to post managed free limit error message: %s" (:error result))
+ (log/errorf "[slackbot] Failed to post pre-flight error message: %s" (:error result))
(throw e)))
(throw e)))))))
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index c21898601b4..0a23a50db8c 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -1073,6 +1073,11 @@
(check! :slackbot {:permission/metabot :no})))
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"permission"
(check! :embedding_next {:permission/metabot :no}))))
+ (testing "the 403 carries only the permission-denied tag and status, so the
+ exception middleware keeps the body a plain message"
+ (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo #"permission"
+ (check! :slackbot {:permission/metabot :no})))]
+ (is (= {:status-code 403 :type :metabot/permission-denied} (ex-data e)))))
(testing "metabot :yes allows non-gated profiles"
(is (nil? (check! :internal {:permission/metabot :yes})))
(is (nil? (check! :slackbot {:permission/metabot :yes})))
diff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.clj
index 49bf5ae0796..72dfc4d0ef9 100644
--- a/test/metabase/metabot/api/document_test.clj
+++ b/test/metabase/metabot/api/document_test.clj
@@ -35,6 +35,15 @@
:post 200 "metabot/document/generate-content"
{:instructions "Show me sales data"}))))))
+(deftest generate-content-permission-denied-body-test
+ (testing "the 403 body is the plain denial sentence, not a map carrying a stack trace"
+ (binding [scope/*current-user-metabot-permissions* {:permission/metabot :yes
+ :permission/metabot-other-tools :no}]
+ (is (= "You do not have permission to use the document-generate-content assistant."
+ (mt/user-http-request :rasta
+ :post 403 "metabot/document/generate-content"
+ {:instructions "Show me sales data"}))))))
+
(deftest generate-content-free-limit-body-test
(testing "the 402 body is the message and error code alone, not a map carrying a stack trace"
(mt/with-dynamic-fn-redefs [metabot.usage/managed-free-limit-reached? (constantly true)]
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 0eb30a5c736..dc7b5b7ab19 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -89,8 +89,9 @@
(#'self/parse-provider-model "metabase/openrouter/anthropic/claude-haiku-4-5"))))
(testing "throws when the string names no configured connection"
(doseq [model-ref ["no-slash" "" "/leading-slash" "nonexistent/some-model"]]
- (is (thrown-with-msg? Exception #"No LLM provider connection named"
- (#'self/parse-provider-model model-ref)))))))
+ (let [e (is (thrown-with-msg? Exception #"No LLM provider connection named"
+ (#'self/parse-provider-model model-ref)))]
+ (is (= :llm-not-configured (:error-code (ex-data e)))))))))
(deftest ^:parallel resolve-adapter-test
(testing "resolves known providers to adapter functions"
diff --git a/test/metabase/slackbot/streaming_test.clj b/test/metabase/slackbot/streaming_test.clj
index 3dbf8d0b3a6..f129f6a5eca 100644
--- a/test/metabase/slackbot/streaming_test.clj
+++ b/test/metabase/slackbot/streaming_test.clj
@@ -7,6 +7,7 @@
[metabase.channel.slack :as channel.slack]
[metabase.metabot.agent.core :as agent]
[metabase.metabot.persistence :as metabot.persistence]
+ [metabase.metabot.scope :as metabot.scope]
[metabase.metabot.settings :as metabot.settings]
[metabase.premium-features.core :as premium-features]
[metabase.slackbot.client :as slackbot.client]
@@ -315,6 +316,108 @@
:text "You've used all of your included AI service tokens. To keep using AI features, end your trial early and start your subscription, or add your own AI provider API key."}
@posted-message))))))
+(defn- dm-error-part-appended-text!
+ "Run a DM turn whose agent loop emits `error-part` instead of text, returning
+ everything appended to the Slack stream."
+ [error-part]
+ (tu/with-slackbot-setup
+ (let [event-body tu/base-dm-event]
+ (tu/with-slackbot-mocks
+ {}
+ (fn [{:keys [append-text-calls stop-stream-calls]}]
+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop
+ (fn [_opts]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (rf init error-part))))
+ metabot.persistence/start-turn!
+ (fn [& _] {:assistant-msg-id 1 :assistant-external-id "ext"})
+ metabot.persistence/finalize-assistant-turn!
+ (fn [& _] nil)]
+ (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)
+ (u/poll {:thunk #(pos? (count @stop-stream-calls))
+ :done? true?
+ :timeout-ms 5000})
+ (str/join "\n" @append-text-calls)))))))
+
+(deftest ^:synchronized slackbot-streamed-error-part-uses-known-error-copy-test
+ (testing "a permission_denied error part becomes access copy, not the raw permission keyword"
+ (let [text (dm-error-part-appended-text!
+ {:type :error :error {:message "Permission denied: :permission/metabot-nlq required"
+ :error-code "permission_denied"}})]
+ (is (str/includes? text "You do not have permission to use the AI assistant."))
+ (is (not (str/includes? text ":permission/")))))
+ (testing "a permission throw the agent loop caught mid-stream also becomes access copy"
+ (let [text (dm-error-part-appended-text!
+ {:type :error :error {:message "Permission denied"
+ :type "clojure.lang.ExceptionInfo"
+ :data {:type :metabot/permission-denied
+ :required-permission :permission/metabot-nlq}}})]
+ (is (str/includes? text "You do not have permission to use the AI assistant."))))
+ (testing "a provider config error the agent loop caught becomes check-your-settings copy"
+ (let [text (dm-error-part-appended-text!
+ {:type :error :error {:message "No LLM provider connection named \"anthropic\" is configured."
+ :type "clojure.lang.ExceptionInfo"
+ :data {:status-code 400 :api-error true :error-code :llm-not-configured}}})]
+ (is (str/includes? text "The AI provider isn't configured correctly. Ask your Metabase admin to check the AI settings."))))
+ (testing "an unrecognized error keeps the generic copy and leaks nothing from the provider"
+ (let [text (dm-error-part-appended-text!
+ {:type :error :error {:message "upstream rejected key sk-ant-oops"}})]
+ (is (str/includes? text "Something went wrong. Please try again."))
+ (is (not (str/includes? text "sk-ant-oops"))))))
+
+(deftest ^:synchronized slackbot-dm-posts-permission-copy-when-metabot-access-denied-test
+ (testing "the 403 the agent loop's access check throws reaches the DM as access copy"
+ (let [run-agent-loop (mt/original-fn #'agent/run-agent-loop)]
+ (tu/with-slackbot-setup
+ (let [event-body tu/base-dm-event]
+ (tu/with-slackbot-mocks
+ {}
+ (fn [{:keys [append-text-calls stop-stream-calls]}]
+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop run-agent-loop
+ metabot.scope/resolve-user-permissions
+ (constantly {:permission/metabot :no})
+ metabot.persistence/start-turn!
+ (fn [& _] {:assistant-msg-id 1 :assistant-external-id "ext"})
+ metabot.persistence/finalize-assistant-turn!
+ (fn [& _] nil)]
+ (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)
+ (u/poll {:thunk #(pos? (count @stop-stream-calls))
+ :done? true?
+ :timeout-ms 5000})
+ (let [text (str/join "\n" @append-text-calls)]
+ (is (str/includes? text "You do not have permission to use the AI assistant."))
+ (is (not (str/includes? text "Something went wrong"))))))))))))
+
+(deftest ^:synchronized slackbot-channel-posts-permission-copy-when-metabot-access-denied-test
+ (testing "the visible channel reply flow posts access copy for the 403, not the generic line"
+ (let [run-agent-loop (mt/original-fn #'agent/run-agent-loop)]
+ (tu/with-slackbot-setup
+ (let [event-body tu/base-mention-event]
+ (tu/with-slackbot-mocks
+ {}
+ (fn [{:keys [post-calls]}]
+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop run-agent-loop
+ metabot.scope/resolve-user-permissions
+ (constantly {:permission/metabot :no})
+ metabot.persistence/start-turn!
+ (fn [& _] {:assistant-msg-id 1 :assistant-external-id "ext"})
+ metabot.persistence/finalize-assistant-turn!
+ (fn [& _] nil)]
+ (mt/client :post 200 "metabot/slack/events"
+ (tu/slack-request-options event-body)
+ event-body)
+ (u/poll {:thunk #(pos? (count @post-calls))
+ :done? true?
+ :timeout-ms 5000})
+ (let [texts (keep :text @post-calls)]
+ (is (some #{"You do not have permission to use the AI assistant."} texts))
+ (is (not-any? #(str/includes? % "Something went wrong") texts)))))))))))
+
(deftest ^:synchronized slackbot-streaming-sets-ai-proxied-on-messages-test
(testing "start-turn! receives ai-proxy? = true (and writes it to both user and assistant rows)
for metabase/ prefixed provider"