Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add missing query recovery hints, drop ones that can't work (#84847)

Andrei Simionescu4d64af0b2ccf
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 78b01407c30..12907f7fd8d 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -209,6 +209,7 @@
metabase.metabot.tools.error/unrecoverable!
metabase.metabot.tools.error-test/test-custom-status!
metabase.metabot.tools.error-test/test-no-widget!
+ metabase.metabot.tools.recoverable.pipeline-test/pipeline-error-text!
metabase.metabot.tools.recoverable.pipeline-test/text-for!
metabase.metabot.tools.recoverable.pipeline-test/two-surface-error!
metabase.metabot.tools.recoverable.pipeline/unknown-table!
diff --git a/src/metabase/mcp/v2/recovery_hints.clj b/src/metabase/mcp/v2/recovery_hints.clj
index 9203d354627..d32b59cf6f8 100644
--- a/src/metabase/mcp/v2/recovery_hints.clj
+++ b/src/metabase/mcp/v2/recovery_hints.clj
@@ -78,7 +78,7 @@
(message/msg [(str "Call \"browse_data\" with action \"list_tables\" for the database, "
"then retry with the numeric table id — it is never ambiguous.")])
- (:unknown-field :unknown-field-id)
+ (:unknown-field :unknown-field-id :ambiguous-field)
(message/msg [(str "Call \"browse_data\" with action \"get_fields\" for "
"the table to list its columns with their numeric ids.")])
@@ -110,7 +110,12 @@
(message/msg [(str "Call \"browse_data\" with action \"get_fields\" for the table that "
"owns the segment and use the numeric id from its segments list.")])
- :unknown-database
+ (:cross-database-card :cross-database-measure :cross-database-segment)
+ (message/msg [(str "A query can only reference saved questions, models, metrics, measures and segments "
+ "from its own database. Use one from that database, or change the source to a table "
+ "in the referenced item's database.")])
+
+ (:unknown-database :ambiguous-database-name)
(message/msg [(str "Use a numeric table id in `source-table:` (from \"browse_data\" "
"action \"list_tables\"), which needs no database name at all.")])
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 91a0aaa9706..a739a5e9404 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -581,9 +581,7 @@
:error :unknown-database
:database db-name}))
1 (first ids)
- (throw (ex-info (tru (str "Multiple databases share the name `{0}` (ids: {1}). The "
- "agent has no way to disambiguate; ask the user to rename "
- "one of the databases or use a more specific identifier.")
+ (throw (ex-info (tru "Multiple databases share the name `{0}` (ids: {1})."
db-name (pr-str (vec (sort ids))))
{:agent-error? true
:status-code 400
diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.clj
index 0afc96c1228..cd7fd684f4f 100644
--- a/src/metabase/metabot/tools/recoverable/pipeline.clj
+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj
@@ -95,6 +95,12 @@
:text (str "`source-table:` takes a portable FK `[<db-name>, <schema>, <table>]`; `source-card:` takes an "
"entity_id string.")})
+(def ^:private cross-database-step
+ {:uses #{}
+ :text (str "A query can only reference saved questions, models, metrics, measures and segments "
+ "from its own database. Use one from that database, or change the source to a table "
+ "in the referenced item's database.")})
+
(defn- card-fields-step
"Return a recovery step listing `what` from the source card, or nil when `payload` names no card."
[{:keys [source-card source-card-type]} what]
@@ -151,7 +157,8 @@
(defpipeline-error ambiguous-database-name!
"The database name in a portable FK matched more than one database."
- [database-name-step])
+ [{:uses #{}
+ :text "Ask the user to rename one of the databases."}])
;;; Field resolution
@@ -165,7 +172,9 @@
(defpipeline-error ambiguous-field!
"A field reference matched more than one column."
- [list-fields-step])
+ [{:uses #{}
+ :text (str "The field name matches more than one column in this table, so it cannot be referenced by name. "
+ "Tell the user about the ambiguous column name.")}])
(defpipeline-error invalid-field-fk!
"A field's portable FK was not a well-formed `[db, schema, table, field]` path."
@@ -231,14 +240,13 @@
[card-entity-id-step])
(defpipeline-error missing-card-entity-id!
- "A card reference carried no entity_id."
- [card-entity-id-step])
+ "The referenced card has no stored `entity_id`, so the query cannot be exported."
+ ;; The stored card has no portable ID for `read_resource` to return.
+ [])
(defpipeline-error cross-database-card!
"The referenced card belongs to a different database than the query."
- ;; No step: the fix is to pick a different source entirely, and which one depends on the question
- ;; the agent is answering, not on anything it can look up.
- [])
+ [cross-database-step])
(defpipeline-error unknown-measure!
"A measure entity_id matched no measure."
@@ -249,12 +257,13 @@
[measure-entity-id-step])
(defpipeline-error missing-measure-entity-id!
- "A measure reference carried no entity_id."
- [measure-entity-id-step])
+ "The referenced measure has no stored `entity_id`, so the query cannot be exported."
+ ;; The stored measure has no portable ID for `read_resource` to return.
+ [])
(defpipeline-error cross-database-measure!
"The referenced measure belongs to a different database than the query."
- [])
+ [cross-database-step])
(defpipeline-error unknown-segment!
"A segment entity_id matched no segment."
@@ -265,12 +274,13 @@
[segment-entity-id-step])
(defpipeline-error missing-segment-entity-id!
- "A segment reference carried no entity_id."
- [segment-entity-id-step])
+ "The referenced segment has no stored `entity_id`, so the query cannot be exported."
+ ;; The stored segment has no portable ID for `read_resource` to return.
+ [])
(defpipeline-error cross-database-segment!
"The referenced segment belongs to a different database than the query."
- [])
+ [cross-database-step])
(defpipeline-error incompatible-metric!
"A metric was used with an incompatible stage source."
diff --git a/src/metabase/metabot/tools/recovery_hints.clj b/src/metabase/metabot/tools/recovery_hints.clj
index 39f227e7936..2e700023e61 100644
--- a/src/metabase/metabot/tools/recovery_hints.clj
+++ b/src/metabase/metabot/tools/recovery_hints.clj
@@ -76,6 +76,9 @@
(:unknown-field :unknown-field-id)
(tru "Call `read_resource` with `metabase://table/<numeric id>/fields` to list this table''s columns.")
+ :ambiguous-field
+ (tru "The field name matches more than one column in this table, so it cannot be referenced by name. Tell the user about the ambiguous column name.")
+
:ambiguous-fk
(or (card-fields-hint data "foreign-key columns")
(tru "Call `read_resource` with `metabase://table/<numeric id>/fields` for the source table to list the available foreign-key columns."))
@@ -95,9 +98,15 @@
(:unknown-segment :unknown-segment-id)
(tru "Do not invent or guess entity_ids: read the table that owns the segment with `read_resource` (`metabase://table/<numeric id>`) and copy the exact `portable_entity_id` from its `<segment>` tag.")
+ (:cross-database-card :cross-database-measure :cross-database-segment)
+ (tru "A query can only reference saved questions, models, metrics, measures and segments from its own database. Use one from that database, or change the source to a table in the referenced item''s database.")
+
:unknown-database
(tru "Use the exact database name as reported by search / `read_resource` (it appears as the first element of every portable FK, e.g. `source-table: [<db-name>, <schema>, <table>]`).")
+ :ambiguous-database-name
+ (tru "Ask the user to rename one of the databases.")
+
:missing-source-in-first-stage
(tru "`source-table:` takes a portable FK `[<db-name>, <schema>, <table>]`; `source-card:` takes an entity_id string.")
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index 560e6fa5009..d7dd637275c 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -14,11 +14,13 @@
[metabase.api.macros.defendpoint.tools-manifest :as tools-manifest]
[metabase.api.macros.scope :as api.scope]
[metabase.collections.models.collection :as collection]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib-be.db :as lib-be.db]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.normalize :as lib.normalize]
+ [metabase.lib.test-util :as lib.tu]
[metabase.mcp.core :as mcp]
[metabase.metabot.agent.profiles :as metabot.profiles]
[metabase.metabot.scope :as metabot.scope]
@@ -327,6 +329,19 @@
:aggregation [(orders-field-ref "ID")])})]
(is (=? {:error "aggregation-entry-not-aggregation"} response)))))
+(deftest construct-query-ambiguous-field-recovery-test
+ (let [mp (mt/metadata-provider)
+ ambiguous-mp (lib.tu/merged-mock-metadata-provider
+ mp {:fields [(assoc (lib.metadata/field mp (mt/id :orders :total)) :id Integer/MAX_VALUE)]})]
+ (mt/with-dynamic-fn-redefs [lib-be/application-database-metadata-provider (constantly ambiguous-mp)]
+ (let [response (mt/user-http-request :rasta :post 400 "agent/v2/construct-query"
+ {:query (orders-query :fields [(orders-field-ref "TOTAL")])})]
+ (is (= "ambiguous-field" (:error response)))
+ (is (str/includes? (:message response)
+ (str "The field name matches more than one column in this table, "
+ "so it cannot be referenced by name. "
+ "Tell the user about the ambiguous column name.")))))))
+
(deftest construct-query-metadata-lookup-failure-test
(mt/with-prometheus-system! [_ system]
(let [query (orders-query :fields [(orders-field-ref "ID")])]
diff --git a/test/metabase/mcp/v2/recovery_hints_test.clj b/test/metabase/mcp/v2/recovery_hints_test.clj
new file mode 100644
index 00000000000..212812c22a0
--- /dev/null
+++ b/test/metabase/mcp/v2/recovery_hints_test.clj
@@ -0,0 +1,25 @@
+(ns metabase.mcp.v2.recovery-hints-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.mcp.v2.message :as message]
+ [metabase.mcp.v2.recovery-hints :as recovery-hints]))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:parallel ambiguous-field-hint-test
+ (is (= (str "Call \"browse_data\" with action \"get_fields\" for "
+ "the table to list its columns with their numeric ids.")
+ (some-> (recovery-hints/recovery-hint {:error :ambiguous-field}) message/render))))
+
+(deftest ^:parallel cross-database-hint-test
+ (doseq [error [:cross-database-card :cross-database-measure :cross-database-segment]]
+ (testing error
+ (is (= (str "A query can only reference saved questions, models, metrics, measures and segments "
+ "from its own database. Use one from that database, or change the source to a table "
+ "in the referenced item's database.")
+ (some-> (recovery-hints/recovery-hint {:error error}) message/render))))))
+
+(deftest ^:parallel ambiguous-database-name-hint-test
+ (is (= (str "Use a numeric table id in `source-table:` (from \"browse_data\" "
+ "action \"list_tables\"), which needs no database name at all.")
+ (some-> (recovery-hints/recovery-hint {:error :ambiguous-database-name}) message/render))))
diff --git a/test/metabase/mcp/v2/tools/query_test.clj b/test/metabase/mcp/v2/tools/query_test.clj
index 204cb5b101b..889bac0269a 100644
--- a/test/metabase/mcp/v2/tools/query_test.clj
+++ b/test/metabase/mcp/v2/tools/query_test.clj
@@ -928,6 +928,20 @@
(is (str/includes? msg "aggregation"))
(is (not (str/includes? msg "portable_entity_id")))))))
+(deftest ^:parallel ambiguous-database-name-recovery-test
+ (let [db-name "query-test duplicate name"
+ query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table [db-name "PUBLIC" "ORDERS"]}]}]
+ (mt/with-temp [:model/Database {db-1 :id} {:name db-name}
+ :model/Database {db-2 :id} {:name db-name}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (testing "the pipeline only states the clash, so v2's numeric-id hint is the one way out"
+ (is (= (str "\"Multiple databases share the name `" db-name "` (ids: " [db-1 db-2] ").\"\n"
+ "Use a numeric table id in `source-table:` (from \"browse_data\" action \"list_tables\"), "
+ "which needs no database name at all.")
+ (error-text (registry/call-tool execute-scope (str (random-uuid)) "execute_query"
+ {:query query})))))))))
+
;;; --------------------------------------- Schema failures and dialect round-trips --------------------------------
;;; GHY-4313. `:query` stays an open `[:map]` at the registry boundary on purpose: the dialect
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index 262fb75ce29..bd428504a94 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -403,6 +403,28 @@
(testing "source-field is populated with ORDERS.PRODUCT_ID (102)"
(is (= 102 (:source-field field-opts)))))))))
+(deftest ambiguous-field-recovery-hint-test
+ (with-mp-and-stubs!
+ (fn []
+ (let [ambiguous-mp (lib.tu/mock-metadata-provider
+ {:database (lib.metadata/database mp)
+ :tables [(lib.metadata/table mp 10)]
+ :fields (conj (vec (lib.metadata/fields mp 10))
+ (assoc (lib.metadata/field mp 101) :id 103))})]
+ (mt/with-dynamic-fn-redefs [lib-be/application-database-metadata-provider (constantly ambiguous-mp)]
+ (let [e (try
+ (construct/execute-representations-query
+ (query-data {"lib/type" "mbql/query"
+ "stages" [{"source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "fields" [["field" {} ["Sample" "PUBLIC" "ORDERS" "TOTAL"]]]}]})
+ {:recovery-hint v1-hints/recovery-hint})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (=? {:error :ambiguous-field :agent-error? true} (ex-data e)))
+ (is (str/ends-with? (ex-message e)
+ (str "The field name matches more than one column in this table, "
+ "so it cannot be referenced by name. "
+ "Tell the user about the ambiguous column name.")))))))))
+
(deftest implicit-join-ambiguous-surfaces-agent-error-test
(with-ambiguous-mp-and-stubs!
(fn []
diff --git a/test/metabase/metabot/tools/recoverable/pipeline_test.clj b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
index 34e503b56be..363767a3e47 100644
--- a/test/metabase/metabot/tools/recoverable/pipeline_test.clj
+++ b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
@@ -249,14 +249,13 @@
(is (str/includes? (text-for! #{"browse_data"}) "portable FK")
"the dialect-specific, tool-free step reaches the numeric-id surface")))
-(deftest ^:parallel the-two-hint-tables-cover-the-same-codes-test
- (testing "the per-surface hint tables answer the same set of codes, which is why one declaration
- per code is the right shape for them. A code one table answers and the other does not
- would mean the surfaces disagree about what is recoverable."
+(deftest ^:parallel shared-recovery-codes-have-hints-on-both-surfaces-test
+ (testing "Metabot and MCP have hints for the shared recovery codes, which are also declared in the catalog"
(let [codes [:uri-in-source-table :unknown-table :unknown-table-id :ambiguous-table
- :unknown-field :unknown-field-id :ambiguous-fk :no-fk-path
+ :unknown-field :unknown-field-id :ambiguous-field :ambiguous-fk :no-fk-path
:unknown-card :unknown-card-id :unknown-measure :unknown-measure-id
- :unknown-segment :unknown-segment-id :unknown-database
+ :unknown-segment :unknown-segment-id :unknown-database :ambiguous-database-name
+ :cross-database-card :cross-database-measure :cross-database-segment
:missing-source-in-first-stage]]
(doseq [code codes]
(testing code
@@ -270,3 +269,30 @@
(is (some? (v2-hints/recovery-hint data)))))))
(testing "and every one of them is declared here"
(is (= #{} (set/difference (set codes) (declared-codes))))))))
+
+(defn- pipeline-error-text!
+ [error]
+ (let [code (keyword declaration-ns (name error))
+ e (try
+ (tools.error/throw-recoverable! code {:message "Query failed."})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (tools.error/recoverable-text (tools.error/classify e) #{})))
+
+(deftest ^:parallel ambiguous-field-recovery-test
+ (is (= (str "Query failed.\n"
+ "The field name matches more than one column in this table, so it cannot be referenced by name. "
+ "Tell the user about the ambiguous column name.")
+ (pipeline-error-text! :ambiguous-field))))
+
+(deftest ^:parallel cross-database-recovery-test
+ (doseq [error [:cross-database-card :cross-database-measure :cross-database-segment]]
+ (testing error
+ (is (= (str "Query failed.\n"
+ "A query can only reference saved questions, models, metrics, measures and segments "
+ "from its own database. Use one from that database, or change the source to a table "
+ "in the referenced item's database.")
+ (pipeline-error-text! error))))))
+
+(deftest ^:parallel ambiguous-database-name-recovery-test
+ (is (= "Query failed.\nAsk the user to rename one of the databases."
+ (pipeline-error-text! :ambiguous-database-name))))
diff --git a/test/metabase/metabot/tools/recovery_hints_test.clj b/test/metabase/metabot/tools/recovery_hints_test.clj
new file mode 100644
index 00000000000..f14ee6eab5b
--- /dev/null
+++ b/test/metabase/metabot/tools/recovery_hints_test.clj
@@ -0,0 +1,23 @@
+(ns metabase.metabot.tools.recovery-hints-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.metabot.tools.recovery-hints :as recovery-hints]))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:parallel ambiguous-field-hint-test
+ (is (= (str "The field name matches more than one column in this table, so it cannot be referenced by name. "
+ "Tell the user about the ambiguous column name.")
+ (recovery-hints/recovery-hint {:error :ambiguous-field}))))
+
+(deftest ^:parallel cross-database-hint-test
+ (doseq [error [:cross-database-card :cross-database-measure :cross-database-segment]]
+ (testing error
+ (is (= (str "A query can only reference saved questions, models, metrics, measures and segments "
+ "from its own database. Use one from that database, or change the source to a table "
+ "in the referenced item's database.")
+ (recovery-hints/recovery-hint {:error error}))))))
+
+(deftest ^:parallel ambiguous-database-name-hint-test
+ (is (= "Ask the user to rename one of the databases."
+ (recovery-hints/recovery-hint {:error :ambiguous-database-name}))))