The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Test that Metabot's recovery hints don't use MCP's vocabulary (#85063)
Andrei Simionescu564b7921c749
diff --git a/test/metabase/metabot/tools/recoverable/pipeline_test.clj b/test/metabase/metabot/tools/recoverable/pipeline_test.cljindex 61c84d5a941..34e503b56be 100644--- a/test/metabase/metabot/tools/recoverable/pipeline_test.clj+++ b/test/metabase/metabot/tools/recoverable/pipeline_test.clj@@ -9,8 +9,11 @@[clojure.set :as set][clojure.string :as str][clojure.test :refer :all]+ [metabase.mcp.v2.message :as message][metabase.mcp.v2.recovery-hints :as v2-hints]+ [metabase.metabot.agent.profiles :as profiles][metabase.metabot.tools.error :as tools.error]+ [metabase.metabot.tools.legacy :as tools.legacy][metabase.metabot.tools.recoverable.pipeline :as pipeline][metabase.metabot.tools.recovery-hints :as v1-hints]))@@ -79,6 +82,103 @@(is (< 20 (count (pipeline-codes))))(is (contains? (pipeline-codes) :unknown-table))))+(def ^:private pipeline-payloads+ "Payloads that reach every branch of the v1 recovery text.++ [[metabase.metabot.tools.recoverable.pipeline/uri-in-source-table!]] and the Agent API's hint for the same code+ pick their sentence by `:entity-type`. The card hints need `:source-card` and `:source-card-type`, and the+ structural hint reads `:paths` in Metabot and `:humanized` in the Agent API."+ (concat+ (for [entity-type [nil "metric" "question" "model" "card" "table"]]+ (cond-> {:message "Query failed."}+ entity-type (assoc :entity-type entity-type :entity-id 1)))+ (for [card-type ["model" "question"]]+ {:message "Query failed." :source-card 12 :source-card-type card-type})+ [{:message "Invalid query."+ :paths "stages[0].filters[0]: invalid type"+ :humanized {"stages" [{"filters" ["invalid type"]}]}}]))++(defn- v1-recovery-steps+ "Each Metabot recovery step paired with its pipeline error code."+ []+ (for [[code {:keys [build-fn]}] (tools.error/recoverables)+ :when (= declaration-ns (namespace code))+ payload pipeline-payloads+ step (:recovery (build-fn payload))]+ [code step]))++(defn- v1-recovery-texts+ "Every recovery sentence that Metabot or the Agent API can send for a pipeline error.++ Metabot's are the steps that its pipeline declarations build, and the Agent API's are the hints that it appends."+ []+ (concat+ (map (comp :text second) (v1-recovery-steps))+ (for [code (pipeline-codes)+ payload pipeline-payloads+ :let [hint (v1-hints/recovery-hint (assoc payload :error code))]+ :when hint]+ hint)))++(defn- mcp-vocabulary+ "The part of `text` that only MCP v2's recovery text uses, or nil.++ That is the `browse_data` or `get_content` tool, or a sentence that mentions a numeric id with `source-table:` or+ `source-card:`. A `<numeric id>` placeholder inside a `metabase://` resource URI only names the resource to read,+ so it doesn't count."+ [text]+ (or (re-find #"browse_data|get_content" text)+ (some #(when (and (re-find #"(?i)\bnumeric\b" %) (re-find #"source-(table|card)" %)) %)+ (str/split (str/replace text #"metabase://[^\s`]*<numeric id>" "") #"(?<=\.)\s+"))))++(deftest ^:parallel v1-recovery-text-keeps-to-v1-vocabulary-test+ (testing "Metabot and the Agent API write portable FKs and entity_ids, so a recovery sentence that names+ `browse_data` or `get_content` or asks for a numeric id would send their agent the wrong way"+ (doseq [text (v1-recovery-texts)]+ (is (nil? (mcp-vocabulary text)) text))))++(deftest ^:parallel v1-recovery-steps-use-v1-tools-test+ (let [tool-names (into #{}+ (map (comp :name tools.legacy/declaration-of))+ (mapcat :tools (vals @@#'profiles/*profiles)))]+ (is (seq tool-names))+ (doseq [[code {:keys [uses]}] (distinct (v1-recovery-steps))]+ (is (set/subset? uses tool-names)+ (str code " uses tools unavailable to Metabot: " (pr-str (set/difference uses tool-names)))))))++(deftest ^:parallel payload-dependent-recovery-text-is-checked-test+ (testing "the payloads reach the card and structural branches on both surfaces, so the vocabulary check+ above reads their text"+ (doseq [[code fragments] {:column-not-returned ["metabase://model/12" "metabase://question/12"]+ :unexportable-fk ["metabase://model/12" "metabase://question/12"]+ :ambiguous-fk ["metabase://model/12" "metabase://question/12"]+ :invalid-representations-query ["stages[0].filters"]}+ :let [{:keys [build-fn]} (get (tools.error/recoverables) (keyword declaration-ns (name code)))+ metabot (for [payload pipeline-payloads, step (:recovery (build-fn payload))] (:text step))+ agent-api (keep #(v1-hints/recovery-hint (assoc % :error code)) pipeline-payloads)]+ [surface texts] {"Metabot" metabot, "the Agent API" agent-api}+ fragment fragments]+ (is (some #(str/includes? % fragment) texts)+ (str surface "'s text for " code " never mentions " fragment)))))++(deftest ^:parallel the-vocabulary-check-finds-only-mcp-advice-test+ (testing "MCP v2's own hints trip it, so the test above can't pass vacuously"+ (doseq [data (concat [{:error :ambiguous-fk} {:error :missing-source-in-first-stage}]+ (for [code [:column-not-returned :unexportable-fk :ambiguous-fk]+ card-type ["model" "question"]]+ {:error code :source-card 12 :source-card-type card-type}))]+ (is (some? (mcp-vocabulary (message/render (v2-hints/recovery-hint data))))+ (pr-str data))))+ (testing "a numeric id placeholder outside a resource URI trips it"+ (doseq [text ["Put `<numeric id>` into `source-card:`."+ "Use `<numeric id>` as `source-table:`."]]+ (is (some? (mcp-vocabulary text)) text)))+ (testing "a placeholder inside a resource URI and a call to `search` don't"+ (doseq [text ["Read `metabase://table/<numeric id>` and copy its portable FK into `source-table:`."+ "Read `metabase://question/<numeric id>` and copy its entity_id into `source-card:`."+ "Call `search` to find the portable FK for `source-table:`."]]+ (is (nil? (mcp-vocabulary text)) text))))+(deftest ^:parallel structural-paths-payload-test(doseq [[paths expected-class] [["stages[0].filters: invalid type" :recoverable][nil :unrecoverable]