Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Name the real table or database in Metabot's recovery steps

Andrei Simionescuf3267bb651e9
diff --git a/src/metabase/metabot/tools/core.clj b/src/metabase/metabot/tools/core.clj
index 41d0cf6366e..744edd1ffd0 100644
--- a/src/metabase/metabot/tools/core.clj
+++ b/src/metabase/metabot/tools/core.clj
@@ -471,20 +471,15 @@
(try
(thunk)
(catch Throwable e
- (let [{:keys [agent-error? status-code error entity-type entity-id humanized
- source-card source-card-type]} (ex-data e)
+ (let [{:keys [agent-error? status-code error humanized]} (ex-data e)
code (pipeline-error-code error)]
(cond
;; Lib exceptions can also carry `:agent-error?`, but their messages aren't written for the model.
(and agent-error? code)
(tools.error/throw-recoverable!
code
- (cond-> {:message (or (ex-message e) "")}
- (some? humanized) (assoc :paths (repr/humanized->paths humanized))
- entity-type (assoc :entity-type entity-type)
- (some? entity-id) (assoc :entity-id entity-id)
- source-card (assoc :source-card source-card
- :source-card-type source-card-type))
+ (cond-> (recoverable.pipeline/payload error e)
+ (some? humanized) (assoc :paths (repr/humanized->paths humanized)))
{:cause e})
(contains? refusal-status-codes status-code)
diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.clj
index cd7fd684f4f..3def00dd7f2 100644
--- a/src/metabase/metabot/tools/recoverable/pipeline.clj
+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj
@@ -19,7 +19,10 @@
gets no steps at all: a missing step reads as terse, a wrong one sends the agent at the wrong
tool."
(:require
+ [clojure.string :as str]
+ [metabase.metabot.db :as metabot.db]
[metabase.metabot.tools.error :refer [defrecoverable]]
+ [metabase.models.interface :as mi]
[metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -31,20 +34,66 @@
the model reads. `:entity-type` and `:entity-id` are present only for the codes whose recovery
depends on what was referenced (see [[uri-in-source-table!]]).
`:source-card` and `:source-card-type` are the numeric ID and type of the stage's `source-card:`,
- present only for the codes a `source-card:` stage raises (see [[column-not-returned!]])."
+ present only for the codes a `source-card:` stage raises (see [[column-not-returned!]]).
+ `:table-id` and `:database-id` are present only for the codes whose recovery steps name that
+ table or database (see [[payload]])."
[:map {:closed true}
[:message :string]
[:paths {:optional true} :string]
[:entity-type {:optional true} [:maybe :string]]
[:entity-id {:optional true} [:maybe [:or :string :int]]]
[:source-card {:optional true} [:maybe :int]]
- [:source-card-type {:optional true} [:maybe [:enum "model" "question"]]]])
+ [:source-card-type {:optional true} [:maybe [:enum "model" "question"]]]
+ [:table-id {:optional true} pos-int?]
+ [:database-id {:optional true} pos-int?]])
+
+(defn readable-database-id
+ "The `:database-id` of the pipeline error `data`, when the current user can read that database.
+
+ Never returns a routing destination's id."
+ [data]
+ (let [id (:database-id data)]
+ (when (and (pos-int? id)
+ ;; `read_resource` 404s on a destination even when the user can read its router.
+ (empty? (metabot.db/destination-database-ids #{id}))
+ (mi/can-read? :model/Database id))
+ id)))
+
+(def ^:private id-keys
+ "For each code whose recovery steps name a table or a database, a map from the payload key to the
+ function that reads that id from the `ex-data`."
+ {:unknown-table {:database-id readable-database-id}
+ :ambiguous-table {:database-id readable-database-id}
+ :unknown-field {:table-id :table-id}
+ :ambiguous-fk {:table-id :source-table}})
+
+(defn payload
+ "The [[::payload]] for the pipeline exception `e`, whose `:error` code is `error`.
+
+ A database id is kept only when [[readable-database-id]] returns it."
+ [error e]
+ (let [{:keys [entity-type entity-id source-card source-card-type] :as data} (ex-data e)
+ {:keys [table-id database-id]} (update-vals (id-keys error) #(% data))]
+ (cond-> {:message (or (ex-message e) "")}
+ entity-type (assoc :entity-type entity-type)
+ (some? entity-id) (assoc :entity-id entity-id)
+ source-card (assoc :source-card source-card
+ :source-card-type source-card-type)
+ ;; The pipeline checks permissions on the query's tables, but finds its database by name first.
+ (pos-int? table-id) (assoc :table-id table-id)
+ database-id (assoc :database-id database-id))))
;;; ------------------------------------------- Shared recovery steps ----------------------------------------------
;; Each step names the tools its text mentions in `:uses`, so a profile without `read_resource` gets
;; the error with that step dropped rather than advice it cannot follow.
+(defn- with-id
+ "`step` with the `<numeric id>` in its text replaced by `id`, or unchanged when `id` is nil."
+ [step id]
+ (cond-> step
+ id (update :text str/replace "<numeric id>" (str id))))
+
(def ^:private list-tables-step
{:uses #{"read_resource"}
:text (str "Call `read_resource` with `metabase://database/<numeric id>/tables` to list the available "
@@ -125,9 +174,12 @@
;;; Table resolution
-(defpipeline-error unknown-table!
+(defrecoverable unknown-table!
"A portable FK `[db, schema, table]` in `source-table:` matched no active table."
- [list-tables-step])
+ {:payload ::payload}
+ [{:keys [message database-id]}]
+ {:message message
+ :recovery [(with-id list-tables-step database-id)]})
(defpipeline-error unknown-table-id!
"A numeric table id did not resolve."
@@ -139,11 +191,15 @@
"`metabase://database/<numeric id>/tables` to find the table, then use its portable FK "
"`[<db-name>, <schema>, <table-name>]` in `source-table:`.")}])
-(defpipeline-error ambiguous-table!
+(defrecoverable ambiguous-table!
"A portable FK matched more than one table."
- [{:uses #{"read_resource"}
- :text (str "Call `read_resource` with `metabase://database/<numeric id>/tables` to list the available "
- "tables and retry with a more specific portable FK.")}])
+ {:payload ::payload}
+ [{:keys [message database-id]}]
+ {:message message
+ :recovery [(-> {:uses #{"read_resource"}
+ :text (str "Call `read_resource` with `metabase://database/<numeric id>/tables` to list the "
+ "available tables and retry with a more specific portable FK.")}
+ (with-id database-id))]})
;;; Database resolution
@@ -162,9 +218,12 @@
;;; Field resolution
-(defpipeline-error unknown-field!
+(defrecoverable unknown-field!
"A field reference named no column of its source."
- [list-fields-step])
+ {:payload ::payload}
+ [{:keys [message table-id]}]
+ {:message message
+ :recovery [(with-id list-fields-step table-id)]})
(defpipeline-error unknown-field-id!
"A numeric field id did not resolve."
@@ -206,9 +265,9 @@
(defrecoverable ambiguous-fk!
"More than one foreign key connects the two tables, so the join path is ambiguous."
{:payload ::payload}
- [payload]
- {:message (:message payload)
- :recovery [(or (card-fields-step payload "foreign-key columns") list-fks-step)]})
+ [{:keys [message table-id] :as payload}]
+ {:message message
+ :recovery [(or (card-fields-step payload "foreign-key columns") (with-id list-fks-step table-id))]})
(defrecoverable unexportable-fk!
"The only foreign key that reaches a field's table could not be resolved, so the field cannot be reached implicitly."
diff --git a/src/metabase/models/serialization/resolve/mp.clj b/src/metabase/models/serialization/resolve/mp.clj
index 9873ad26470..c9e826e514e 100644
--- a/src/metabase/models/serialization/resolve/mp.clj
+++ b/src/metabase/models/serialization/resolve/mp.clj
@@ -117,12 +117,13 @@
The LLM can still self-correct in one turn by listing the parent database's tables with
the calling surface's discovery tool; the message points it at that path."
- [_metadata-provider [_path-db-name _path-schema _path-table-name :as path]]
+ [metadata-provider [_path-db-name _path-schema _path-table-name :as path]]
(ex-info (tru "No table found matching portable FK {0}." (pr-str path))
{:status-code 400
:error :unknown-table
:agent-error? true
- :path path}))
+ :path path
+ :database-id (:id (lib.metadata/database metadata-provider))}))
(defn- table-candidates
"Resolve `(schema, table-name)` against `metadata-provider`, returning only active tables.
@@ -156,6 +157,9 @@
re-hallucinating the same bad path. All error branches are marked
`:agent-error? true` so the outer tool wrapper relays the message verbatim.
+ Every error here carries the provider's `:database-id`, which is not permission-checked: read it
+ with `metabase.metabot.tools.recoverable.pipeline/readable-database-id`.
+
Inactive (`:active false`) tables are dropped by [[table-candidates]], so a stale FK to one
surfaces here as a 0-candidate miss."
[metadata-provider [path-db-name path-schema path-table-name :as path]]
@@ -168,7 +172,8 @@
:error :unknown-table
:agent-error? true
:path path
- :expected-db current-db})))
+ :expected-db current-db
+ :database-id current-db-id})))
(let [candidates (table-candidates metadata-provider current-db-id path-schema path-table-name)]
(case (count candidates)
0 (throw (unknown-table-ex-info metadata-provider path))
@@ -181,7 +186,8 @@
{:status-code 400
:error :ambiguous-table
:agent-error? true
- :path path}))))))
+ :path path
+ :database-id current-db-id}))))))
(defn- find-field
"Resolve a field path `[db schema table field …]` to a `:metadata/column` or throw with context.
diff --git a/test/metabase/metabot/tools/core_test.clj b/test/metabase/metabot/tools/core_test.clj
index f8b12cb68bf..459873128a7 100644
--- a/test/metabase/metabot/tools/core_test.clj
+++ b/test/metabase/metabot/tools/core_test.clj
@@ -369,7 +369,7 @@
{:agent-error? true :error error :source-card 12 :source-card-type "model"}))))))
(testing "an ambiguous foreign key on a `source-table:` stage still points at the table's fields"
(is (=? {:recovery [{:uses #{"read_resource"}
- :text (str "Call `read_resource` with `metabase://table/<numeric id>/fields` for the source "
+ :text (str "Call `read_resource` with `metabase://table/3/fields` for the source "
"table to list the available foreign-key columns.")}]}
(pipeline-failure "Two foreign keys reach the table."
{:agent-error? true :error :ambiguous-fk :source-table 3})))))
diff --git a/test/metabase/metabot/tools/recoverable/pipeline_test.clj b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
index 363767a3e47..39653a67581 100644
--- a/test/metabase/metabot/tools/recoverable/pipeline_test.clj
+++ b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
@@ -9,13 +9,20 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
+ [clojure.walk :as walk]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.test-util :as lib.tu]
[metabase.mcp.v2.message :as message]
[metabase.mcp.v2.recovery-hints :as v2-hints]
[metabase.metabot.agent.profiles :as profiles]
+ [metabase.metabot.test-util :as test-util]
+ [metabase.metabot.tools.construct :as construct]
[metabase.metabot.tools.error :as tools.error]
[metabase.metabot.tools.legacy :as tools.legacy]
[metabase.metabot.tools.recoverable.pipeline :as pipeline]
- [metabase.metabot.tools.recovery-hints :as v1-hints]))
+ [metabase.metabot.tools.recovery-hints :as v1-hints]
+ [metabase.metabot.tools.resources :as resources]
+ [metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -179,6 +186,102 @@
"Call `search` to find the portable FK for `source-table:`."]]
(is (nil? (mcp-vocabulary text)) text))))
+;;; ===== Recovery steps name the real table or database =====
+
+(defn- construct-failure
+ "Run `construct_notebook_query` on a one-stage test-data query that fails, and return the failure
+ as a profile with `read_resource` reads it. `stage` is string-keyed, as the model writes it."
+ [stage]
+ (test-util/tool-failure construct/construct-notebook-query-tool
+ {:query (walk/keywordize-keys
+ {"lib/type" "mbql/query"
+ "stages" [(merge {"lib/type" "mbql.stage/mbql"} stage)]})
+ :title "Orders"
+ :description "Orders by month."}
+ #{"read_resource"}))
+
+(defn- recovery-uri
+ "The `metabase://` URI that a recovery step tells the agent to read."
+ [text]
+ (second (re-find #"`(metabase://[^`]+)`" text)))
+
+(defn- read-resource
+ [uri]
+ (test-util/call-tool resources/read-resource-tool {:uris [uri]} #{"read_resource"}))
+
+(deftest recovery-steps-name-the-real-table-or-database-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [db (:name (mt/db))
+ tables-uri (str "metabase://database/" (mt/id) "/tables")
+ orders-uri (str "metabase://table/" (mt/id :orders) "/fields")
+ check-failure (fn [stage code uri]
+ (let [failure (construct-failure stage)]
+ (is (= code (:code failure)))
+ (is (= uri (recovery-uri (:text failure))))
+ (testing "and `read_resource` reads that URI"
+ (is (some? (:structured-output (read-resource uri)))))))]
+ (testing "an unknown table names the query's database"
+ (check-failure {"source-table" [db "PUBLIC" "ORDRS"]
+ "aggregation" [["count" {}]]}
+ :metabase.metabot.tools.recoverable.pipeline/unknown-table
+ tables-uri))
+ (testing "an ambiguous table names the query's database"
+ ;; The app DB's unique keys forbid a duplicate table, so both tables exist only in the metadata provider
+ (let [mp (lib.tu/mock-metadata-provider
+ (mt/metadata-provider)
+ {:tables [{:id 99999901 :db-id (mt/id) :schema "PUBLIC" :name "SHIPMENTS"}
+ {:id 99999902 :db-id (mt/id) :schema "PUBLIC" :name "SHIPMENTS"}]})]
+ (mt/with-dynamic-fn-redefs [lib-be/application-database-metadata-provider (constantly mp)]
+ (check-failure {"source-table" [db "PUBLIC" "SHIPMENTS"]
+ "aggregation" [["count" {}]]}
+ :metabase.metabot.tools.recoverable.pipeline/ambiguous-table
+ tables-uri))))
+ (testing "an unknown column names its table"
+ (check-failure {"source-table" [db "PUBLIC" "ORDERS"]
+ "aggregation" [["count" {}]]
+ "breakout" [["field" {} [db "PUBLIC" "ORDERS" "CREATD_AT"]]]}
+ :metabase.metabot.tools.recoverable.pipeline/unknown-field
+ orders-uri))
+ (testing "an unknown column on another table names that table"
+ (check-failure {"source-table" [db "PUBLIC" "ORDERS"]
+ "aggregation" [["count" {}]]
+ "breakout" [["field" {} [db "PUBLIC" "PRODUCTS" "CATEGRY"]]]}
+ :metabase.metabot.tools.recoverable.pipeline/unknown-field
+ (str "metabase://table/" (mt/id :products) "/fields")))
+ (testing "an ambiguous foreign key names the source table"
+ (mt/with-temp-vals-in-db :model/Field (mt/id :orders :user_id) {:fk_target_field_id (mt/id :products :id)}
+ (check-failure {"source-table" [db "PUBLIC" "ORDERS"]
+ "aggregation" [["count" {}]]
+ "breakout" [["field" {} [db "PUBLIC" "PRODUCTS" "CATEGORY"]]]}
+ :metabase.metabot.tools.recoverable.pipeline/ambiguous-fk
+ orders-uri))))))
+
+(deftest recovery-steps-name-the-database-only-when-the-user-can-read-it-test
+ (let [stage {"source-table" [(:name (mt/db)) "PUBLIC" "ORDRS"]
+ "aggregation" [["count" {}]]}]
+ (testing "the database is found by name before any permission check, so a user who cannot read it
+ gets the placeholder instead of its id"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [{:keys [code text]} (construct-failure stage)]
+ (is (= :metabase.metabot.tools.recoverable.pipeline/unknown-table code))
+ (is (= "metabase://database/<numeric id>/tables" (recovery-uri text)))))))
+ (testing "a user who can read it gets its id"
+ (mt/with-full-data-perms-for-all-users!
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [{:keys [text]} (construct-failure stage)]
+ (is (= (str "metabase://database/" (mt/id) "/tables") (recovery-uri text)))))))))
+
+(deftest recovery-steps-do-not-name-a-routing-destination-test
+ (testing "`read_resource` never opens a routing destination, so even an admin gets the placeholder"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Database {router-id :id} {}
+ :model/Database {destination :name} {:router_database_id router-id}]
+ (let [{:keys [code text]} (construct-failure {"source-table" [destination "PUBLIC" "ORDERS"]
+ "aggregation" [["count" {}]]})]
+ (is (= :metabase.metabot.tools.recoverable.pipeline/unknown-table code))
+ (is (= "metabase://database/<numeric id>/tables" (recovery-uri text))))))))
+
(deftest ^:parallel structural-paths-payload-test
(doseq [[paths expected-class] [["stages[0].filters: invalid type" :recoverable]
[nil :unrecoverable]