Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Test that Metabot's recovery hints don't use MCP's vocabulary (#85063)

Andrei Simionescu564b7921c749
diff --git a/test/metabase/metabot/tools/recoverable/pipeline_test.clj b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
index 61c84d5a941..34e503b56be 100644
--- a/test/metabase/metabot/tools/recoverable/pipeline_test.clj
+++ b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
@@ -9,8 +9,11 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.mcp.v2.message :as message]
[metabase.mcp.v2.recovery-hints :as v2-hints]
+ [metabase.metabot.agent.profiles :as profiles]
[metabase.metabot.tools.error :as tools.error]
+ [metabase.metabot.tools.legacy :as tools.legacy]
[metabase.metabot.tools.recoverable.pipeline :as pipeline]
[metabase.metabot.tools.recovery-hints :as v1-hints]))
@@ -79,6 +82,103 @@
(is (< 20 (count (pipeline-codes))))
(is (contains? (pipeline-codes) :unknown-table))))
+(def ^:private pipeline-payloads
+ "Payloads that reach every branch of the v1 recovery text.
+
+ [[metabase.metabot.tools.recoverable.pipeline/uri-in-source-table!]] and the Agent API's hint for the same code
+ pick their sentence by `:entity-type`. The card hints need `:source-card` and `:source-card-type`, and the
+ structural hint reads `:paths` in Metabot and `:humanized` in the Agent API."
+ (concat
+ (for [entity-type [nil "metric" "question" "model" "card" "table"]]
+ (cond-> {:message "Query failed."}
+ entity-type (assoc :entity-type entity-type :entity-id 1)))
+ (for [card-type ["model" "question"]]
+ {:message "Query failed." :source-card 12 :source-card-type card-type})
+ [{:message "Invalid query."
+ :paths "stages[0].filters[0]: invalid type"
+ :humanized {"stages" [{"filters" ["invalid type"]}]}}]))
+
+(defn- v1-recovery-steps
+ "Each Metabot recovery step paired with its pipeline error code."
+ []
+ (for [[code {:keys [build-fn]}] (tools.error/recoverables)
+ :when (= declaration-ns (namespace code))
+ payload pipeline-payloads
+ step (:recovery (build-fn payload))]
+ [code step]))
+
+(defn- v1-recovery-texts
+ "Every recovery sentence that Metabot or the Agent API can send for a pipeline error.
+
+ Metabot's are the steps that its pipeline declarations build, and the Agent API's are the hints that it appends."
+ []
+ (concat
+ (map (comp :text second) (v1-recovery-steps))
+ (for [code (pipeline-codes)
+ payload pipeline-payloads
+ :let [hint (v1-hints/recovery-hint (assoc payload :error code))]
+ :when hint]
+ hint)))
+
+(defn- mcp-vocabulary
+ "The part of `text` that only MCP v2's recovery text uses, or nil.
+
+ That is the `browse_data` or `get_content` tool, or a sentence that mentions a numeric id with `source-table:` or
+ `source-card:`. A `<numeric id>` placeholder inside a `metabase://` resource URI only names the resource to read,
+ so it doesn't count."
+ [text]
+ (or (re-find #"browse_data|get_content" text)
+ (some #(when (and (re-find #"(?i)\bnumeric\b" %) (re-find #"source-(table|card)" %)) %)
+ (str/split (str/replace text #"metabase://[^\s`]*<numeric id>" "") #"(?<=\.)\s+"))))
+
+(deftest ^:parallel v1-recovery-text-keeps-to-v1-vocabulary-test
+ (testing "Metabot and the Agent API write portable FKs and entity_ids, so a recovery sentence that names
+ `browse_data` or `get_content` or asks for a numeric id would send their agent the wrong way"
+ (doseq [text (v1-recovery-texts)]
+ (is (nil? (mcp-vocabulary text)) text))))
+
+(deftest ^:parallel v1-recovery-steps-use-v1-tools-test
+ (let [tool-names (into #{}
+ (map (comp :name tools.legacy/declaration-of))
+ (mapcat :tools (vals @@#'profiles/*profiles)))]
+ (is (seq tool-names))
+ (doseq [[code {:keys [uses]}] (distinct (v1-recovery-steps))]
+ (is (set/subset? uses tool-names)
+ (str code " uses tools unavailable to Metabot: " (pr-str (set/difference uses tool-names)))))))
+
+(deftest ^:parallel payload-dependent-recovery-text-is-checked-test
+ (testing "the payloads reach the card and structural branches on both surfaces, so the vocabulary check
+ above reads their text"
+ (doseq [[code fragments] {:column-not-returned ["metabase://model/12" "metabase://question/12"]
+ :unexportable-fk ["metabase://model/12" "metabase://question/12"]
+ :ambiguous-fk ["metabase://model/12" "metabase://question/12"]
+ :invalid-representations-query ["stages[0].filters"]}
+ :let [{:keys [build-fn]} (get (tools.error/recoverables) (keyword declaration-ns (name code)))
+ metabot (for [payload pipeline-payloads, step (:recovery (build-fn payload))] (:text step))
+ agent-api (keep #(v1-hints/recovery-hint (assoc % :error code)) pipeline-payloads)]
+ [surface texts] {"Metabot" metabot, "the Agent API" agent-api}
+ fragment fragments]
+ (is (some #(str/includes? % fragment) texts)
+ (str surface "'s text for " code " never mentions " fragment)))))
+
+(deftest ^:parallel the-vocabulary-check-finds-only-mcp-advice-test
+ (testing "MCP v2's own hints trip it, so the test above can't pass vacuously"
+ (doseq [data (concat [{:error :ambiguous-fk} {:error :missing-source-in-first-stage}]
+ (for [code [:column-not-returned :unexportable-fk :ambiguous-fk]
+ card-type ["model" "question"]]
+ {:error code :source-card 12 :source-card-type card-type}))]
+ (is (some? (mcp-vocabulary (message/render (v2-hints/recovery-hint data))))
+ (pr-str data))))
+ (testing "a numeric id placeholder outside a resource URI trips it"
+ (doseq [text ["Put `<numeric id>` into `source-card:`."
+ "Use `<numeric id>` as `source-table:`."]]
+ (is (some? (mcp-vocabulary text)) text)))
+ (testing "a placeholder inside a resource URI and a call to `search` don't"
+ (doseq [text ["Read `metabase://table/<numeric id>` and copy its portable FK into `source-table:`."
+ "Read `metabase://question/<numeric id>` and copy its entity_id into `source-card:`."
+ "Call `search` to find the portable FK for `source-table:`."]]
+ (is (nil? (mcp-vocabulary text)) text))))
+
(deftest ^:parallel structural-paths-payload-test
(doseq [[paths expected-class] [["stages[0].filters: invalid type" :recoverable]
[nil :unrecoverable]