Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Convert four more Metabot tools to the tool contract (#85037)

Andrei Simionescue66356efd549
diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
index 3c9f6248183..95212c514b7 100644
--- a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
@@ -8,6 +8,7 @@
[metabase-enterprise.search.semantic.embedding :as semantic.embedding]
[metabase-enterprise.search.semantic.embedding-health :as embedding-health]
[metabase-enterprise.search.semantic.test-util :as semantic.tu]
+ [metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.entity-retrieval :as tools.entity-retrieval]
[metabase.search.entity-retrieval.mirror :as mirror]
[metabase.test :as mt]
@@ -434,7 +435,9 @@
;; bind a user: the tool's hydration permission-filters via mi/can-read?, which needs
;; *current-user-id* (a superuser can read the test tables).
search! #(mt/with-test-user :crowberto
- (get-in (tools.entity-retrieval/retrieve-library-entities-tool {:user_search_prompt q})
+ (get-in (tools.core/handle tools.entity-retrieval/retrieve-library-entities-tool
+ {:user_search_prompt q}
+ {})
[:structured-output :data]))]
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
(constantly semantic.tu/mock-embedding-model)]
diff --git a/src/metabase/metabot/tools/autogen_dashboard.clj b/src/metabase/metabot/tools/autogen_dashboard.clj
index b976fc3a85d..bcaeaa705d9 100644
--- a/src/metabase/metabot/tools/autogen_dashboard.clj
+++ b/src/metabase/metabot/tools/autogen_dashboard.clj
@@ -1,5 +1,5 @@
(ns metabase.metabot.tools.autogen-dashboard
- "Autogenerated dashboard tool wrappers."
+ "The `create_autogenerated_dashboard` tool."
(:require
[clojure.string :as str]
[medley.core :as m]
@@ -10,9 +10,9 @@
[metabase.metabot.db :as metabot.db]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.recoverable.common :as recoverable.common]
[metabase.metabot.tools.shared :as shared]
[metabase.util.log :as log]
- [metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.xrays.core :as xrays]))
@@ -79,10 +79,26 @@
(or (pos-int? v)
(and (string? v) (not (str/blank? v)))))
+(defn- normalized-query
+ "`query`, one of the conversation's queries, normalized, or nil when it names no database or query type yet."
+ [query]
+ (when (and (:database query) (lib/normalized-query-type query))
+ (lib-be/normalize-query query)))
+
+(defn- analyzable-query-ids
+ "The IDs of the conversation's `queries` that X-ray can analyze, sorted: the complete queries that don't start
+ with SQL."
+ [queries]
+ (sort (for [[id query] queries
+ :let [mbql-query (normalized-query query)]
+ :when (and (mr/validate :metabase.lib.schema/query mbql-query)
+ (not (lib/native? mbql-query)))]
+ id)))
+
(defn- resolve-source
- "Resolve the source entity from the provided source parameters.
- Returns the entity to analyze and the path for navigation."
- [source memory]
+ "The X-ray path for `source` under `:path`, and the table, card or query that it reads under `:entity`.
+ A `query_id` names one of `queries`, the conversation's queries."
+ [source queries]
(let [{:keys [table_id model_id metric_id query_id report_id] :as source} (m/filter-vals source-value? source)]
(when-not (= 1 (count source))
(invalid-source! {}))
@@ -93,16 +109,12 @@
report_id (readable-source :question report_id)
query_id
- (let [queries-state (get-in memory [:state :queries] {})
- query (get queries-state query_id)
- mbql-query (when (and (:database query) (lib/normalized-query-type query))
- (lib-be/normalize-query query))]
+ (let [query (get queries query_id)
+ mbql-query (normalized-query query)]
(when-not query
- (throw (ex-info (str "Query not found in memory with ID: " query_id
- ". Available queries: [" (str/join ", " (keys queries-state)) "]")
- {:agent-error? true
- :query-id query_id
- :available-queries (keys queries-state)})))
+ (recoverable.common/unknown-query-id! {:query-id query_id
+ :available (analyzable-query-ids queries)
+ :hint :notebook-only}))
(when-not (mr/validate :metabase.lib.schema/query mbql-query)
(incomplete-query! {:query-id query_id}))
(let [entity (tools/with-entity {:kind :query :id query_id}
@@ -112,55 +124,6 @@
{:entity entity
:path (str "/auto/dashboard/adhoc/" (xrays/encode-base64-json query))})))))
-;;; Response Messages
-
-(def ^:private response-messages
- ["Ok, here is a dashboard that may answer it for you."
- "I've created a dashboard that may answer your question."
- "Here is a dashboard that may help you with your question."
- "I've generated a dashboard with insights about your data."])
-
-(defn- random-response-message []
- (rand-nth response-messages))
-
-;;; Main Tool Implementation
-
-(defn create-autogenerated-dashboard
- "Create an auto-generated dashboard using Metabase's X-ray feature.
-
- X-ray builds complete dashboards with multiple visualizations that showcase
- different aspects of your data - distributions, breakdowns by categories,
- trends over time, and relationships between fields.
-
- Parameters:
- - source: Map specifying what to analyze. One of:
- - {:table_id 123} - Create dashboard for a table
- - {:model_id 456} - Create dashboard for a model
- - {:metric_id 789} - Create dashboard for a metric
- - {:query_id \"abc\"} - Create dashboard for notebook query results (SQL queries must be saved first)
- - {:report_id 101} - Create dashboard for an existing report/question
- - memory-atom: Atom containing agent memory (for query resolution)
-
- Returns map with:
- - :structured-output - Message and dashboard info
- - :data-parts - Dashboard `generated_entity` part for the frontend
- - :instructions - Guidance for the LLM on how to respond"
- [{:keys [source memory-atom]}]
- (log/info "Creating autogenerated dashboard" {:source source})
-
- (let [memory (when memory-atom @memory-atom)
- {:keys [entity path]} (resolve-source source memory)
- title (or (:display_name entity) (:name entity) "Dashboard")
- response-message (str (random-response-message) "\n\nLet me know if you need anything else.")]
- ;; Note: The actual automagic analysis happens on navigation to the /auto/dashboard/... URL
- ;; The frontend handles rendering the X-ray dashboard
-
- (log/info "Autogenerated dashboard created" {:source source})
- {:structured-output {:message response-message
- :path path}
- :data-parts [(streaming/dashboard-entity-part {:title title :url path})]
- :instructions "The dashboard has been created and the user has been navigated to view it. Briefly acknowledge this (1 sentence) and stop. Do not describe the dashboard contents."}))
-
(def ^:private autogen-dashboard-schema
[:map {:closed true}
[:source [:map {:closed true}
@@ -170,28 +133,34 @@
[:query_id {:optional true} [:maybe :string]]
[:report_id {:optional true} [:maybe :int]]]]])
-(mu/defn ^{:tool-name "create_autogenerated_dashboard"
- :scope scope/agent-dashboard-create}
- create-autogenerated-dashboard-tool
- "Create an automatically generated dashboard using Metabase's X-ray feature.
-
- X-ray builds complete dashboards with multiple visualizations that showcase
- different aspects of your data - distributions, breakdowns by categories,
- trends over time, and relationships between fields.
-
- Specify one source type:
- - table_id: Create dashboard for a table
- - model_id: Create dashboard for a model
- - metric_id: Create dashboard for a metric
- - query_id: Create dashboard for notebook queries from this conversation (SQL queries must be saved first)
- - report_id: Create dashboard for an existing report/question"
- [{:keys [source]} :- autogen-dashboard-schema]
- (let [result (create-autogenerated-dashboard
- {:source source
- :memory-atom shared/*memory-atom*})
- structured (:structured-output result)
- data-parts (:data-parts result)]
- (cond-> {:output (:message structured)
- :structured-output structured
- :instructions (:instructions result)}
- (seq data-parts) (assoc :data-parts data-parts))))
+(defrecord CreateAutogeneratedDashboardTool []
+ tools/Tool
+ (declaration [_]
+ {:name "create_autogenerated_dashboard"
+ :description (str "Create an automatically generated dashboard using Metabase's X-ray feature.\n\n"
+ "X-ray builds complete dashboards with multiple visualizations that showcase different "
+ "aspects of your data - distributions, breakdowns by categories, trends over time, and "
+ "relationships between fields.\n\n"
+ "Specify one source type:\n"
+ "- table_id: Create dashboard for a table\n"
+ "- model_id: Create dashboard for a model\n"
+ "- metric_id: Create dashboard for a metric\n"
+ "- query_id: Create dashboard for notebook queries from this conversation (SQL queries must "
+ "be saved first)\n"
+ "- report_id: Create dashboard for an existing report/question")
+ :scope scope/agent-dashboard-create
+ :args autogen-dashboard-schema})
+
+ (handle [_ {:keys [source]} {:keys [memory-atom]}]
+ (log/info "Creating autogenerated dashboard" {:source source})
+ ;; The X-ray itself runs when the frontend opens `path`
+ (let [{:keys [entity path]} (resolve-source source (shared/current-queries-state memory-atom))]
+ (log/info "Autogenerated dashboard created" {:source source})
+ {:output (str "The dashboard has been created and the user has been navigated to view it. Briefly "
+ "acknowledge this (1 sentence) and stop. Do not describe the dashboard contents.")
+ :data-parts [(streaming/dashboard-entity-part {:title (or (:display_name entity) (:name entity) "Dashboard")
+ :url path})]})))
+
+(def create-autogenerated-dashboard-tool
+ "The `create_autogenerated_dashboard` tool."
+ (->CreateAutogeneratedDashboardTool))
diff --git a/src/metabase/metabot/tools/entity_retrieval.clj b/src/metabase/metabot/tools/entity_retrieval.clj
index c38be67c381..312bea61706 100644
--- a/src/metabase/metabot/tools/entity_retrieval.clj
+++ b/src/metabase/metabot/tools/entity_retrieval.clj
@@ -17,11 +17,11 @@
(:require
[clojure.string :as str]
[metabase.metabot.scope :as scope]
+ [metabase.metabot.tools.core :as tools]
[metabase.metabot.tools.search :as tools.search]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.search.entity-retrieval.core :as entity-retrieval]
- [metabase.util :as u]
- [metabase.util.malli :as mu]))
+ [metabase.util :as u]))
(set! *warn-on-reflection* true)
@@ -167,29 +167,46 @@
:confidence (if weak? "weak" "strong")))
matches))
+(tools/defrecoverable blank-search-prompt!
+ "`user_search_prompt` is empty or only whitespace."
+ {:payload [:map {:closed true}]}
+ [_]
+ {:message "`user_search_prompt` is blank. Set it to a full sentence that describes the data the user wants."
+ :recovery []})
+
;; No capability gate: this tool lives only in the :nlq profile, which [[metabase.metabot.agent.profiles/
;; get-profile]] serves (the library variant, with this tool) only when entity-retrieval-available? — otherwise it
;; redirects to :nlq-fallback (general search). That single probe is the sole gate, so prompt and tools can't
;; disagree about index availability.
-(mu/defn ^{:tool-name "retrieve_library_entities"
- :scope scope/agent-search}
- retrieve-library-entities-tool
- "Find the best data to answer the user's request from the library — the set of entities published to it
- (published tables, library metrics/models, and their measures/segments). Phrase
- `user_search_prompt` as a full natural-language description of the data wanted (it is matched on
- meaning, not keywords).
-
- Returns a handful of distinct library entities, best-first. Each match has the `matched_on` text that
- triggered it, any curator `usage_instructions`, a raw `similarity`, and the `entity` — a full search
- record (name, type, database, `portable_entity_id`, fully-qualified name). Use it to pick the right
- entity, then `read_resource` that entity to confirm its fields and sample values before building. If the
- top match is flagged low-confidence (a leading <note> / confidence=\"weak\"), nothing in the library
- clearly matches — prefer asking the user to clarify or narrow the request."
- [{:keys [user_search_prompt limit]} :- entity-retrieval-schema]
- (let [n (min max-limit (or limit default-limit))
- matches (build-matches user_search_prompt n)]
- {:output (format-output matches)
- :structured-output {:result-type :search
- :data (flatten-data matches)
- :total_count (count matches)
- :weak_match (boolean (:weak? (first matches)))}}))
+(defrecord RetrieveLibraryEntitiesTool []
+ tools/Tool
+ (declaration [_]
+ {:name "retrieve_library_entities"
+ :description (str "Find the best data to answer the user's request from the library – the set of entities "
+ "published to it (published tables, library metrics/models, and their measures/segments). "
+ "Phrase `user_search_prompt` as a full natural-language description of the data wanted (it "
+ "is matched on meaning, not keywords).\n\n"
+ "Returns a handful of distinct library entities, best-first. Each match has the `matched_on` "
+ "text that triggered it, any curator `usage_instructions`, a raw `similarity`, and the "
+ "`entity` – a full search record (name, type, database, `portable_entity_id`, "
+ "fully-qualified name). Use it to pick the right entity, then `read_resource` that entity to "
+ "confirm its fields and sample values before building. If the top match is flagged "
+ "low-confidence (a leading <note> / confidence=\"weak\"), nothing in the library clearly "
+ "matches – prefer asking the user to clarify or narrow the request.")
+ :scope scope/agent-search
+ :args entity-retrieval-schema})
+
+ (handle [_ {:keys [user_search_prompt limit]} _ctx]
+ (when (str/blank? user_search_prompt)
+ (blank-search-prompt! {}))
+ (let [n (min max-limit (or limit default-limit))
+ matches (build-matches user_search_prompt n)]
+ {:output (format-output matches)
+ :structured-output {:result-type :search
+ :data (flatten-data matches)
+ :total_count (count matches)
+ :weak_match (boolean (:weak? (first matches)))}})))
+
+(def retrieve-library-entities-tool
+ "The `retrieve_library_entities` tool."
+ (->RetrieveLibraryEntitiesTool))
diff --git a/src/metabase/metabot/tools/error.clj b/src/metabase/metabot/tools/error.clj
index bf92129c55a..ca2a571da12 100644
--- a/src/metabase/metabot/tools/error.clj
+++ b/src/metabase/metabot/tools/error.clj
@@ -346,6 +346,19 @@
(cons message)
(str/join "\n")))
+(def max-output-length
+ "Maximum number of characters in a rendered failure, including recovery steps.
+
+ Leave room for recovery steps when including excerpts."
+ 4000)
+
+(defn output-budget
+ "Return the character budget for an excerpt after `prefix` and every recovery step."
+ [prefix recovery]
+ (let [tool-names (into #{} (mapcat :uses) recovery)]
+ (- max-output-length
+ (count (recoverable-text {:message prefix :recovery recovery} tool-names)))))
+
(defn authored-text
"Return `text` with the quoted payload values from `error`'s declaration removed.
diff --git a/src/metabase/metabot/tools/recoverable/common.clj b/src/metabase/metabot/tools/recoverable/common.clj
index 2f4ce8a5847..924c45396ee 100644
--- a/src/metabase/metabot/tools/recoverable/common.clj
+++ b/src/metabase/metabot/tools/recoverable/common.clj
@@ -6,7 +6,7 @@
specific."
(:require
[clojure.string :as str]
- [metabase.metabot.tools.error :refer [defrecoverable]]
+ [metabase.metabot.tools.error :as tools.error :refer [defrecoverable]]
[metabase.util :as u]
[metabase.util.quoting :as quoting]))
@@ -47,18 +47,76 @@
[{:uses #{finder}
:text (str "Call `" finder "` to find the entity you want and use an id from the results.")}])})
+(def ^:private max-query-id-chars
+ "The most characters of the requested query ID that [[unknown-query-id!]] repeats.
+ The model supplies the ID, so its length is otherwise unbounded."
+ 100)
+
+(defn- available-query-ids-text
+ "`ids` quoted and joined within `room` characters: the leading IDs that fit whole, then a count of the rest."
+ [ids room]
+ (let [quoted (mapv quoting/quoted ids)
+ n (count quoted)
+ running-lengths (reductions + (map-indexed (fn [i id] (+ (count id) (if (pos? i) 2 0))) quoted))]
+ (cond
+ (zero? n) "none"
+ (<= (last running-lengths) room) (str/join ", " quoted)
+ :else
+ (let [limit (- room (count (str " and " n " more")))
+ shown (count (take-while #(<= % limit) running-lengths))]
+ (if (pos? shown)
+ (str (str/join ", " (take shown quoted)) " and " (- n shown) " more")
+ (str n ", too long to list"))))))
+
+(defn- unknown-query-id-advice
+ "What [[unknown-query-id!]] tells the model to do for `hint`: a `:sentence` after its message, and its `:recovery`."
+ [hint any-available?]
+ (let [use-available {:uses #{} :text "Use an available query ID from this conversation."}]
+ (case hint
+ nil
+ {:recovery [(if any-available?
+ use-available
+ {:uses #{"create_sql_query"}
+ :text "Use `create_sql_query` to create a SQL query, then use the returned query ID."})]}
+
+ :notebook-only
+ {:recovery [(if any-available?
+ use-available
+ {:uses #{"construct_notebook_query"}
+ :text "Use `construct_notebook_query` to build a query, then use the returned query ID."})]}
+
+ :notebook-query
+ {:sentence " Use the ID of a query that you built, or a saved question or model that the user can read."
+ :recovery []}
+
+ :sql-query
+ {:sentence " A saved question or model has no query ID."
+ :recovery [{:uses #{"create_sql_query" "run_query"}
+ :text (str "To run one, write SQL that reads it with `create_sql_query`, then run that query"
+ " with `run_query`.")}]}
+
+ :no-values
+ {:sentence " A saved question or model has no query ID, and no SQL can run here."
+ :recovery [{:uses #{} :text "Tell the user you can't read its results."}]})))
+
(defrecoverable unknown-query-id!
- "The requested query is absent from the conversation."
+ "The query ID names no query that the tool can use for the current user.
+
+ `:hint` picks the recovery. By default the model uses a listed ID or creates a SQL query, and `:notebook-only`
+ offers a notebook query instead. `run_query` passes one of the others for its profile, since it also runs saved
+ questions by ID."
{:payload [:map {:closed true}
- [:query-id :string]
- [:available [:sequential :string]]]}
- [{:keys [query-id available]}]
- {:message (str "Query " (quoting/quoted query-id) " was not found. Available query IDs: "
- (if (seq available) (str/join ", " (map quoting/quoted available)) "none") ".")
- :recovery [(if (seq available)
- {:uses #{} :text "Use an available query ID from this conversation."}
- {:uses #{"create_sql_query"}
- :text "Use `create_sql_query` to create a SQL query, then use the returned query ID."})]})
+ [:query-id :string]
+ [:available [:sequential :string]]
+ [:hint {:optional true} [:enum :notebook-only :notebook-query :sql-query :no-values]]]}
+ [{:keys [query-id available hint]}]
+ (let [{:keys [sentence recovery]} (unknown-query-id-advice hint (boolean (seq available)))
+ opening (str "Query " (quoting/quoted (u/truncate query-id max-query-id-chars))
+ " was not found. Available query IDs: ")
+ ;; The IDs get only the room left by the rest, so a long list never pushes the advice past the runtime's cut
+ room (tools.error/output-budget (str opening "." sentence) recovery)]
+ {:message (str opening (available-query-ids-text available room) "." sentence)
+ :recovery recovery}))
(defrecoverable unknown-database!
"The requested database does not exist."
diff --git a/src/metabase/metabot/tools/run_query.clj b/src/metabase/metabot/tools/run_query.clj
index 438a9d10e61..ab77ee0008b 100644
--- a/src/metabase/metabot/tools/run_query.clj
+++ b/src/metabase/metabot/tools/run_query.clj
@@ -14,6 +14,7 @@
[metabase.metabot.settings :as metabot.settings]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.recoverable.common :as recoverable.common]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.metabot.tools.sql.common :as sql.common]
@@ -89,46 +90,6 @@
:recovery [{:uses #{}
:text "Ask the user to continue in their own Metabot chat."}]})
-(def ^:private max-listed-query-ids
- "The refusal lists at most this many known query IDs.
-
- This leaves room for a recovery step within the runtime's limit on failure text."
- 100)
-
-(def ^:private max-echoed-query-id-chars
- "The refusal repeats at most this many characters from an unknown query ID.
-
- The model supplies the ID, so its length is otherwise unbounded."
- 100)
-
-(tools/defrecoverable unknown-query-id!
- "The query ID matches no stored query or saved question that this tool can run for the current user."
- {:payload [:map {:closed true}
- [:query-id :string]
- [:known-ids [:sequential :string]]
- [:hint [:enum :notebook-query :sql-query :no-values]]]}
- [{:keys [query-id known-ids hint]}]
- (let [more (- (count known-ids) max-listed-query-ids)
- unknown (str "No query with ID " (llm-shape/truncate query-id max-echoed-query-id-chars) ". Known query IDs: ["
- (str/join ", " (take max-listed-query-ids known-ids)) "]"
- (when (pos? more) (str " and " more " more"))
- ".")]
- (case hint
- :notebook-query
- {:message (str unknown " Use the ID of a query that you built, or a saved question or model that the user"
- " can read.")
- :recovery []}
-
- :sql-query
- {:message (str unknown " A saved question or model has no query ID.")
- :recovery [{:uses #{"create_sql_query" "run_query"}
- :text (str "To run one, write SQL that reads it with `create_sql_query`, then run that query"
- " with `run_query`.")}]}
-
- :no-values
- {:message (str unknown " A saved question or model has no query ID, and no SQL can run here.")
- :recovery [(:no-values hint-steps)]})))
-
(tools/defrecoverable metric-not-runnable!
"The ID names a metric that the user can read, and a metric does not run by its ID.
A metric is an aggregation, not a source, so no query with it as the source would run."
@@ -292,16 +253,17 @@
"Return the query named by `query-id` in a map under `:query`.
Saved questions are available only when `allow-non-sql?` is true and the user can read them;
- the result then includes the card under `:saved-card`. Throw [[unknown-query-id!]] if no query is found."
+ the result then includes the card under `:saved-card`. Throw [[recoverable.common/unknown-query-id!]] if no query
+ is found."
[query-id allow-non-sql?]
(let [queries (shared/current-queries-state)]
(or (when-let [query (get queries query-id)]
{:query query})
(when allow-non-sql?
(saved-question query-id))
- (unknown-query-id! {:query-id query-id
- :known-ids (vec (keys queries))
- :hint (recovery-hint allow-non-sql?)}))))
+ (recoverable.common/unknown-query-id! {:query-id query-id
+ :available (sort (keys queries))
+ :hint (recovery-hint allow-non-sql?)}))))
(defn- check-private-conversation!
"Refuse unless the current conversation belongs to the current user alone.
diff --git a/src/metabase/metabot/tools/runtime.clj b/src/metabase/metabot/tools/runtime.clj
index 77cb3fdc10a..8e0eae2db11 100644
--- a/src/metabase/metabot/tools/runtime.clj
+++ b/src/metabase/metabot/tools/runtime.clj
@@ -28,10 +28,17 @@
[metabase.metabot.tools.shared :as shared]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [potemkin :as p]))
(set! *warn-on-reflection* true)
+;; Declared in `tools.error` so that the declarations this namespace loads can budget against the same limit
+(p/import-vars
+ [tools.error
+ max-output-length
+ output-budget])
+
(def runtime-codes
"Every error code the runtime itself produces, with the class each one takes.
@@ -175,19 +182,6 @@
". Tool errors may only carry authored text; log the exception instead.")
{:where where})))))
-(def max-output-length
- "Maximum number of characters in a rendered failure, including recovery steps.
-
- Leave room for recovery steps when including excerpts."
- 4000)
-
-(defn output-budget
- "Return the character budget for an excerpt after `prefix` and every recovery step."
- [prefix recovery]
- (let [tool-names (into #{} (mapcat :uses) recovery)]
- (- max-output-length
- (count (tools.error/recoverable-text {:message prefix :recovery recovery} tool-names)))))
-
(defn render
"The failure [[::outcome]] for a ToolError, given the tool names available this turn.
diff --git a/src/metabase/metabot/tools/shared.clj b/src/metabase/metabot/tools/shared.clj
index 78694e86c68..0d036cc3c72 100644
--- a/src/metabase/metabot/tools/shared.clj
+++ b/src/metabase/metabot/tools/shared.clj
@@ -49,9 +49,12 @@
(:request-canceled-chan (current-memory)))
(defn current-queries-state
- "Returns the current queries state map from agent memory."
- []
- (get-in (current-memory) [:state :queries] {}))
+ "Returns the current queries state map from agent memory.
+ Reads the memory in `memory-atom`, which a tool gets in its ctx, or else the bound [[*memory-atom*]]."
+ ([]
+ (current-queries-state *memory-atom*))
+ ([memory-atom]
+ (get-in (some-> memory-atom deref) [:state :queries] {})))
(defn current-charts-state
"Returns the current charts state map from agent memory."
diff --git a/src/metabase/metabot/tools/skills.clj b/src/metabase/metabot/tools/skills.clj
index ea342404787..cf2216eefe1 100644
--- a/src/metabase/metabot/tools/skills.clj
+++ b/src/metabase/metabot/tools/skills.clj
@@ -7,34 +7,86 @@
(:require
[clojure.string :as str]
[metabase.metabot.skills :as skills]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]))
+ [metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.util.log :as log]))
(set! *warn-on-reflection* true)
-(defn- load-one
- "Resolve and format a single skill id, or an inline message when it is unknown
- or not available to the current user."
+(def ^:private max-echoed-id-chars
+ "The failure repeats at most this many characters of the requested ID, which the model supplies."
+ 100)
+
+(defn- loadable-skill-ids
+ "The IDs of the skills the current request may load, sorted."
+ []
+ (->> (skills/all-skills)
+ (filter skills/skill-loadable?)
+ (map (comp name :id))
+ sort
+ vec))
+
+(defn- not-loadable-text
+ "The sentence saying that the skill `id` can't be loaded here, without the skills that can."
[id]
- (let [skill (skills/get-skill-by-id-string id)]
- (cond
- (nil? skill)
- (do (log/info "Unknown skill requested by load_skill")
- (format "Unknown skill: \"%s\". Load only ids listed under \"Available skills\"." id))
-
- (not (skills/skill-loadable? skill))
- (do (log/info "Skill not available to current request" {:id id})
- (format "Skill \"%s\" is not available to you." id))
-
- :else
- (format "<skill id=\"%s\">\n%s\n</skill>" (name (:id skill)) (:body skill)))))
-
-(mu/defn ^{:tool-name "load_skill"}
- load-skill-tool
- "Load the full instructions for one or more skills listed under \"Available
- skills\". Call this immediately before doing the related work; you may load
- several skills at once. Load only what is relevant to the current task."
- [{:keys [ids]} :- [:map {:closed true}
- [:ids [:sequential
- [:string {:description "A skill id, e.g. construct-notebook-query-core"}]]]]]
- {:output (str/join "\n\n" (map load-one ids))})
+ (str "No skill with ID " (pr-str (llm-shape/truncate id max-echoed-id-chars)) " can be loaded here."))
+
+(tools/defrecoverable unknown-skill!
+ "No skill has the requested ID, or the current request may not load it."
+ {:payload [:map {:closed true}
+ [:id :string]
+ [:loadable [:sequential :string]]]}
+ [{:keys [id loadable]}]
+ ;; A missing skill and a hidden one get the same sentence, so the reply reveals nothing about hidden skills
+ {:message (str (not-loadable-text id) " "
+ (if (seq loadable)
+ (str "Skill IDs you can load: " (str/join ", " loadable) ".")
+ "No skills can be loaded here."))
+ :recovery []})
+
+(def ^:private skill-id-schema
+ [:string {:description "A skill id, e.g. construct-notebook-query-core"}])
+
+(defrecord LoadSkillTool []
+ tools/Tool
+ (declaration [_]
+ {:name "load_skill"
+ :description (str "Load the full instructions for a skill listed under \"Available skills\". Call this "
+ "immediately before doing the related work. Load only what is relevant to the current task.")
+ :args [:map {:closed true} [:id skill-id-schema]]})
+
+ (handle [_ {:keys [id]} _ctx]
+ (let [skill (skills/get-skill-by-id-string id)]
+ (when-not (some-> skill skills/skill-loadable?)
+ (if skill
+ (log/info "Skill not available to current request" {:id id})
+ (log/info "Unknown skill requested by load_skill"))
+ (unknown-skill! {:id id :loadable (loadable-skill-ids)}))
+ {:output (format "<skill id=\"%s\">\n%s\n</skill>" (name (:id skill)) (:body skill))}))
+
+ tools/BatchedTool
+ (batched-declaration [_ declared]
+ (assoc declared
+ :description (str "Load the full instructions for one or more skills listed under \"Available "
+ "skills\". Call this immediately before doing the related work; you may load several "
+ "skills at once. Load only what is relevant to the current task.")
+ ;; The dialect preload's synthetic call sends `ids` too (`skills/dialect-preload-parts`)
+ :args [:map {:closed true}
+ [:ids [:sequential {:min 1 :error/message "must be an array of at least one skill id"}
+ skill-id-schema]]]))
+
+ (batched-args [_ {:keys [ids]}]
+ (mapv (fn [id] {:id id}) ids))
+
+ (compose [_ entries _ctx]
+ ;; Only the first failure lists the loadable skills, so they appear once and survive the runtime's length limit
+ (let [first-failed (first (keep-indexed (fn [i entry] (when (:failed? entry) i)) entries))]
+ {:output (str/join "\n\n" (map-indexed (fn [i {:keys [failed? output error]}]
+ (if (and failed? (not= i first-failed))
+ (not-loadable-text (get-in error [:data :id]))
+ output))
+ entries))})))
+
+(def load-skill-tool
+ "`load_skill`: one skill through [[tools/handle]], several through [[tools/run-batched]]."
+ (->LoadSkillTool))
diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.clj
index bb24ad3fb93..03a0985f7b6 100644
--- a/src/metabase/metabot/tools/sql.clj
+++ b/src/metabase/metabot/tools/sql.clj
@@ -7,6 +7,7 @@
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.core :as tools]
[metabase.metabot.tools.run-query :as tools.run-query]
+ [metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.metabot.tools.sql.common :as sql.common]
@@ -159,7 +160,7 @@
{:query-id query-id
:edits edits
:checklist checklist
- :queries-state (get-in (some-> (:memory-atom ctx) deref) [:state :queries] {})})
+ :queries-state (shared/current-queries-state (:memory-atom ctx))})
title instructions/edit-sql-query-instructions-for ctx)))
(def edit-sql-query-tool
@@ -191,7 +192,7 @@
{:query-id query-id
:sql new-query
:checklist checklist
- :queries-state (get-in (some-> (:memory-atom ctx) deref) [:state :queries] {})})
+ :queries-state (shared/current-queries-state (:memory-atom ctx))})
title instructions/replace-sql-query-instructions-for ctx)))
(def replace-sql-query-tool
diff --git a/src/metabase/metabot/tools/timelines.clj b/src/metabase/metabot/tools/timelines.clj
index 0913ff1b845..61d5da0a9a1 100644
--- a/src/metabase/metabot/tools/timelines.clj
+++ b/src/metabase/metabot/tools/timelines.clj
@@ -1,31 +1,15 @@
(ns metabase.metabot.tools.timelines
- "`list_timelines` and `get_timeline_details`.
-
- `get_timeline_details` is converted: it is a record implementing
- `metabase.metabot.tools.core/Tool`. `list_timelines` is not yet, so it is still an `mu/defn` var.
- Both appear in the same profile; see `metabase.metabot.tools/->entries`."
+ "`list_timelines` and `get_timeline_details`."
(:require
[clojure.string :as str]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.core :as tools]
[metabase.metabot.tools.recoverable.common :as recoverable.common]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
- [metabase.metabot.tools.util :as metabot.tools.u]
- [metabase.timeline.core :as timeline]
- [metabase.util.malli :as mu]))
+ [metabase.timeline.core :as timeline]))
(set! *warn-on-reflection* true)
-(defn get-timelines
- "Lists timelines visible to the current user."
- [_args]
- (try
- {:structured_output
- (->> (timeline/list-timelines)
- (map #(select-keys % [:id :name :description])))}
- (catch Exception e
- (metabot.tools.u/handle-agent-error e))))
-
(defn get-timeline-details
"A timeline by id, including its events, or nil when there is no such timeline.
@@ -67,22 +51,21 @@
"<events />\n")
"</timeline>"))
-(defn- add-output
- "Add :output to a tool result."
- [result format-fn]
- (if-let [structured (or (:structured_output result) (:structured-output result))]
- (assoc result :output (format-fn structured))
- result))
-
-(mu/defn ^{:tool-name "list_timelines"
- :scope scope/agent-timelines-read}
- list-timelines-tool
- "List all timelines available in the Metabase instance.
+(defrecord ListTimelinesTool []
+ tools/Tool
+ (declaration [_]
+ {:name "list_timelines"
+ :description (str "List all timelines available in the Metabase instance.\n\n"
+ "If a timeline looks relevant to the user's request, fetch its events using the "
+ "get_timeline_details tool.")
+ :scope scope/agent-timelines-read
+ :args [:map {:closed true}]})
+ (handle [_ _args _ctx]
+ {:output (format-timeline-list-output (timeline/list-timelines))}))
- If a timeline looks relevant to the user's request, fetch its events using the
- get_timeline_details tool."
- [_args :- [:map {:closed true}]]
- (add-output (get-timelines {}) format-timeline-list-output))
+(def list-timelines-tool
+ "The `list_timelines` tool."
+ (->ListTimelinesTool))
(defrecord GetTimelineDetailsTool []
tools/Tool
diff --git a/test/metabase/metabot/tools/autogen_dashboard_test.clj b/test/metabase/metabot/tools/autogen_dashboard_test.clj
index cf97e6b1f44..85c6bd98290 100644
--- a/test/metabase/metabot/tools/autogen_dashboard_test.clj
+++ b/test/metabase/metabot/tools/autogen_dashboard_test.clj
@@ -2,108 +2,48 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.api-scope.core :as api-scope]
[metabase.api.macros :as api.macros]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.agent.core :as agent]
[metabase.metabot.context :as metabot.context]
- [metabase.metabot.test-util :as test-util]
+ [metabase.metabot.db :as metabot.db]
+ [metabase.metabot.scope :as scope]
+ [metabase.metabot.tools :as metabot.tools]
[metabase.metabot.tools.autogen-dashboard :as autogen-dashboard]
- [metabase.metabot.tools.legacy :as tools.legacy]
- [metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.recoverable.common :as recoverable.common]
[metabase.test :as mt]
[metabase.util.log.capture :as log.capture]))
-;;; Source Resolution Tests (without database)
+(defn- invoke
+ "Call `create_autogenerated_dashboard` with `source` the way the agent loop does, in a conversation that holds
+ `queries`, in a profile that also has `other-tools` (`search` by default). Return the result, or `{:output <the
+ text the model reads> :error {:class :code}}` when the call fails."
+ ([source] (invoke source {}))
+ ([source queries] (invoke source queries [#'metabot.tools/search-tool]))
+ ([source queries other-tools]
+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]
+ (metabot.tools/->entries (cons #'metabot.tools/create-autogenerated-dashboard-tool other-tools)
+ (atom {:state {:queries queries}})
+ nil
+ :internal))]
+ ((:fn (get entries "create_autogenerated_dashboard")) {:source source}))))
-(deftest resolve-source-validation-test
- (testing "fails when no source specified"
- (let [memory-atom (atom {:state {:queries {} :charts {}}})]
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Set exactly one of"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {}
- :memory-atom memory-atom})))))
- (testing "fails when query_id not found in memory"
- (let [memory-atom (atom {:state {:queries {}}})]
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Query not found in memory"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {:query_id "nonexistent"}
- :memory-atom memory-atom}))))))
+(defn- dashboard
+ "The result of a call that created the dashboard titled `title` at `url`."
+ [title url]
+ {:output (str "The dashboard has been created and the user has been navigated to view it. Briefly "
+ "acknowledge this (1 sentence) and stop. Do not describe the dashboard contents.")
+ :data-parts [{:type :data
+ :data-type "generated_entity"
+ :data {:type "dashboard" :url url :title title}}]})
-;;; Response Structure Tests
-
-(deftest response-structure-test
- (mt/with-current-user (mt/user->id :rasta)
- (testing "returns instructions for LLM"
- ;; We need to test the structure without actually hitting the database
- ;; Using mt/with-temp for table creation
- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id (:id table)}
- :memory-atom (atom {:state {}})})]
- (is (string? (:instructions result))))))
- (testing "returns a dashboard generated_entity data part"
- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id (:id table)}
- :memory-atom (atom {:state {}})})
- part (first (:data-parts result))
- entity (:data part)]
- (is (contains? result :data-parts))
- (is (= "generated_entity" (:data-type part)))
- (is (= "dashboard" (:type entity)))
- (is (str/starts-with? (:url entity) "/auto/dashboard/"))
- (is (= (or (:display_name table) (:name table)) (:title entity))))))
- (testing "returns user-friendly message"
- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id (:id table)}
- :memory-atom (atom {:state {}})})]
- (is (string? (get-in result [:structured-output :message])))
- (is (str/includes? (get-in result [:structured-output :message]) "dashboard")))))
- (testing "returns path in structured output"
- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id (:id table)}
- :memory-atom (atom {:state {}})})]
- (is (string? (get-in result [:structured-output :path]))))))))
-
-;;; Path Generation Tests
-
-(deftest path-generation-test
- (mt/with-current-user (mt/user->id :rasta)
- (testing "table source generates correct path"
- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id (:id table)}
- :memory-atom (atom {:state {}})})]
- (is (= (str "/auto/dashboard/table/" (:id table))
- (get-in result [:structured-output :path]))))))
- (testing "model source generates correct path"
- (mt/with-temp [:model/Card model {:type :model :name "test_model"}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:model_id (:id model)}
- :memory-atom (atom {:state {}})})]
- (is (= (str "/auto/dashboard/model/" (:id model))
- (get-in result [:structured-output :path]))))))
- (testing "metric source generates correct path"
- (mt/with-temp [:model/Card metric {:type :metric :name "test_metric"}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:metric_id (:id metric)}
- :memory-atom (atom {:state {}})})]
- (is (= (str "/auto/dashboard/question/" (:id metric))
- (get-in result [:structured-output :path]))))))
- (testing "report source generates correct path"
- (mt/with-temp [:model/Card report {:type :question :name "test_report"}]
- (let [result (autogen-dashboard/create-autogenerated-dashboard
- {:source {:report_id (:id report)}
- :memory-atom (atom {:state {}})})]
- (is (= (str "/auto/dashboard/question/" (:id report))
- (get-in result [:structured-output :path]))))))))
+(defn- failure
+ "A call that failed with the declared error `code`, and the `lines` that the model reads."
+ [code & lines]
+ {:output (str/join "\n" lines)
+ :error {:class :recoverable :code code}})
(defn- xray-page
"Fetch the dashboard for `path` from the X-ray API as `:rasta`."
@@ -115,17 +55,37 @@
(let [mp (mt/metadata-provider)]
(lib/query mp (lib.metadata/table mp (mt/id :venues)))))
+(deftest dashboard-for-each-source-test
+ (mt/with-temp [:model/Table {table-id :id :as table} {:name (mt/random-name) :db_id (mt/id)}
+ :model/Card {model-id :id} {:type :model :name "Test model"}
+ :model/Card {metric-id :id} {:type :metric :name "Test metric"}
+ :model/Card {report-id :id} {:type :question :name "Test report"}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (testing "the model is told to acknowledge the dashboard in a sentence, and the frontend gets a link to it"
+ (is (= (dashboard (or (:display_name table) (:name table)) (str "/auto/dashboard/table/" table-id))
+ (invoke {:table_id table-id}))))
+ (testing "each kind of source has its own X-ray path"
+ (is (= (dashboard "Test model" (str "/auto/dashboard/model/" model-id))
+ (invoke {:model_id model-id})))
+ (is (= (dashboard "Test metric" (str "/auto/dashboard/question/" metric-id))
+ (invoke {:metric_id metric-id})))
+ (is (= (dashboard "Test report" (str "/auto/dashboard/question/" report-id))
+ (invoke {:report_id report-id}))))
+ (testing "a query from the conversation is analyzed as it is"
+ (is (=? (dashboard "Dashboard" #"^/auto/dashboard/adhoc/.+")
+ (invoke {:query_id "q123"} {"q123" (venues-query)})))))))
+
(deftest xray-page-test
(let [venues (venues-query)]
(mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query venues}
:model/Card {metric-id :id} {:type :metric :dataset_query (lib/aggregate venues (lib/count))}
:model/Card {question-id :id} {:type :question :dataset_query venues}]
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" venues
- "q-legacy" (lib/->legacy-MBQL venues)
- "q-model" (lib/query (mt/metadata-provider)
- (lib.metadata/card
- (mt/metadata-provider) model-id))
- "q-staged" (lib/append-stage venues)}}})]
+ (let [queries {"q-1" venues
+ "q-legacy" (lib/->legacy-MBQL venues)
+ "q-model" (lib/query (mt/metadata-provider)
+ (lib.metadata/card
+ (mt/metadata-provider) model-id))
+ "q-staged" (lib/append-stage venues)}]
(mt/with-current-user (mt/user->id :rasta)
(doseq [source [{:table_id (mt/id :venues)}
{:model_id model-id}
@@ -137,69 +97,78 @@
{:query_id "q-staged"}]]
(testing (str "the X-ray link for " (pr-str source) " returns a dashboard from the API")
(is (=? {:dashcards seq}
- (->> (autogen-dashboard/create-autogenerated-dashboard-tool {:source source})
- :structured-output
- :path
+ (->> (invoke source queries)
+ :data-parts first :data :url
(xray-page 200)))))))))))
(deftest log-test
(testing "the log records the source and leaves out the query text"
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" (venues-query)}}})]
+ (let [queries {"q-1" (venues-query)}]
(log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.autogen-dashboard :info]]
(mt/with-current-user (mt/user->id :rasta)
- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-1"}}))
+ (invoke {:query_id "q-1"} queries))
(is (= ["Creating autogenerated dashboard {:source {:query_id q-1}}"
"Autogenerated dashboard created {:source {:query_id q-1}}"]
(map :message (logs))))))))
-;;; Entity Not Found Tests
+(deftest unknown-query-id-test
+ (testing "a query ID that the conversation doesn't hold fails as it does for the other tools that take one"
+ (is (= (failure ::recoverable.common/unknown-query-id
+ "Query \"q-9\" was not found. Available query IDs: \"q-1\"."
+ "Use an available query ID from this conversation.")
+ (invoke {:query_id "q-9"} {"q-1" (venues-query)}))))
+ (testing "with no query to use, the model is told to build a notebook query, since X-ray refuses a SQL one"
+ (is (= (failure ::recoverable.common/unknown-query-id
+ "Query \"q-9\" was not found. Available query IDs: none."
+ "Use `construct_notebook_query` to build a query, then use the returned query ID.")
+ (invoke {:query_id "q-9"} {} [#'metabot.tools/construct-notebook-query-tool
+ #'metabot.tools/create-sql-query-tool]))))
+ (testing "only the queries that X-ray can analyze are listed, not the SQL or incomplete ones that it refuses"
+ (let [sql (lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES")]
+ (is (= (failure ::recoverable.common/unknown-query-id
+ "Query \"q-9\" was not found. Available query IDs: \"q-1\", \"q-legacy\"."
+ "Use an available query ID from this conversation.")
+ (invoke {:query_id "q-9"} {"q-1" (venues-query)
+ "q-legacy" (lib/->legacy-MBQL (venues-query))
+ "q-sql" sql
+ "q-sql-legacy" (lib/->legacy-MBQL sql)
+ "q-sql-staged" (lib/append-stage sql)
+ "q-draft" {:database (mt/id)}})))))
+ (testing "a long list of IDs is cut to complete IDs, so the recovery step still reaches the model"
+ (let [{:keys [output error]} (invoke {:query_id "q-9"}
+ (zipmap (repeatedly 100 #(str (random-uuid))) (repeat (venues-query))))]
+ (is (= {:class :recoverable :code ::recoverable.common/unknown-query-id} error))
+ (is (re-find #"\" and \d+ more\.\nUse an available query ID from this conversation\.$" output)))))
-(deftest entity-not-found-test
- (testing "fails for nonexistent table"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Table 999999 was not found"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {:table_id 999999}
- :memory-atom (atom {:state {}})}))))
- (testing "fails for nonexistent model"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Model 999999 was not found"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {:model_id 999999}
- :memory-atom (atom {:state {}})}))))
- (testing "fails for nonexistent metric"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Metric 999999 was not found"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {:metric_id 999999}
- :memory-atom (atom {:state {}})}))))
- (testing "fails for nonexistent report"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Question 999999 was not found"
- (autogen-dashboard/create-autogenerated-dashboard
- {:source {:report_id 999999}
- :memory-atom (atom {:state {}})})))))
-
-(defn- failure
- "How a `create_autogenerated_dashboard` call with `source` fails, as the agent sees it."
- [source]
- (test-util/tool-failure (tools.legacy/adapt #'autogen-dashboard/create-autogenerated-dashboard-tool)
- {:source source}))
+(deftest one-source-test
+ (testing "a call that sets no source, or two, is rejected before any source is read"
+ (doseq [source [{} {:table_id 1 :model_id 2}]]
+ (is (= (failure ::autogen-dashboard/invalid-source
+ "Set exactly one of `table_id`, `model_id`, `metric_id`, `query_id` or `report_id` in `source`.")
+ (invoke source)))))
+ (testing "a placeholder for an unused source is not a second source"
+ (mt/with-temp [:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id)}
+ :model/Card {model-id :id} {:type :model}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (doseq [[source path] [[{:table_id nil :model_id model-id} (str "/auto/dashboard/model/" model-id)]
+ [{:table_id table-id :model_id 0} (str "/auto/dashboard/table/" table-id)]
+ [{:table_id 0 :model_id model-id} (str "/auto/dashboard/model/" model-id)]
+ [{:query_id "" :table_id table-id} (str "/auto/dashboard/table/" table-id)]]]
+ (testing (pr-str source)
+ (is (=? {:data-parts [{:data {:url path}}]}
+ (invoke source)))))))))
-(deftest create-autogenerated-dashboard-tool-errors-test
- (testing "a query missing from memory goes back to the agent as a failed call"
- (is (= {:class :recoverable
- :code :metabase.metabot.tools.recoverable.legacy/agent-error
- :text "Query not found in memory with ID: q-9. Available queries: []"}
- (failure {:query_id "q-9"}))))
- (testing "an unexpected error propagates to the agent loop"
- (mt/with-dynamic-fn-redefs [autogen-dashboard/create-autogenerated-dashboard (fn [_] (throw (ex-info "boom" {})))]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:table_id 1}}))))))
+(deftest missing-source-test
+ (mt/with-current-user (mt/user->id :rasta)
+ (doseq [[source-key noun] {:table_id "Table"
+ :model_id "Model"
+ :metric_id "Metric"
+ :report_id "Question"}]
+ (testing noun
+ (is (= (failure ::recoverable.common/not-found
+ (str noun " 999999 was not found. It may not exist, or you may not have access to it.")
+ "Call `search` to find the entity you want and use an id from the results.")
+ (invoke {source-key 999999})))))))
(deftest unreadable-source-test
(mt/with-non-admin-groups-no-root-collection-perms
@@ -215,32 +184,30 @@
[:table_id table-id "Table"]]]
(testing noun
(testing "fails for a user who cannot read it, the same way a missing one does"
- (is (=? {:class :recoverable
- :code :metabase.metabot.tools.recoverable.common/not-found
- :text (re-pattern (str "(?s)" noun " " id " was not found\\..*"))}
- (mt/with-current-user (mt/user->id :rasta)
- (failure {source-key id})))))
+ (is (= (failure ::recoverable.common/not-found
+ (str noun " " id " was not found. It may not exist, or you may not have access to it.")
+ "Call `search` to find the entity you want and use an id from the results.")
+ (mt/with-current-user (mt/user->id :rasta)
+ (invoke {source-key id})))))
(testing "still gets an admin a dashboard"
(is (=? {:data-parts [{:data {:title #"Secret .*"}}]}
(mt/with-current-user (mt/user->id :crowberto)
- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {source-key id}})))))))))))
+ (invoke {source-key id})))))))))))
(deftest unrunnable-query-test
(testing "a user without query permissions gets no X-ray link"
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" (venues-query)}}})]
- (let [path (mt/with-current-user (mt/user->id :crowberto)
- (-> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-1"}})
- :structured-output
- :path))]
- (mt/with-no-data-perms-for-all-users!
- (is (= "You don't have permissions to do that."
- (xray-page 403 path)))
- (is (= {:class :recoverable
- :code :metabase.metabot.tools.recoverable.common/not-found
- :text (str "Query q-1 was not found. It may not exist, or you may not have access to it.\n"
- "Call `search` to find the entity you want and use an id from the results.")}
- (mt/with-current-user (mt/user->id :rasta)
- (failure {:query_id "q-1"})))))))))
+ (let [queries {"q-1" (venues-query)}
+ path (mt/with-current-user (mt/user->id :crowberto)
+ (->> (invoke {:query_id "q-1"} queries)
+ :data-parts first :data :url))]
+ (mt/with-no-data-perms-for-all-users!
+ (is (= "You don't have permissions to do that."
+ (xray-page 403 path)))
+ (is (= (failure ::recoverable.common/not-found
+ "Query q-1 was not found. It may not exist, or you may not have access to it."
+ "Call `search` to find the entity you want and use an id from the results.")
+ (mt/with-current-user (mt/user->id :rasta)
+ (invoke {:query_id "q-1"} queries))))))))
(deftest unreadable-joined-card-test
(mt/with-non-admin-groups-no-root-collection-perms
@@ -252,20 +219,19 @@
query (-> (venues-query)
(lib/join (lib.metadata/card mp card-id)))]
(doseq [query [query (lib/->legacy-MBQL query)]]
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-join" query}}})]
- (let [path (mt/with-current-user (mt/user->id :crowberto)
- (-> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-join"}})
- :structured-output :path))]
- (testing "the X-ray API denies a query joining a card the user cannot read"
- (is (= "You don't have permissions to do that."
- (xray-page 403 path))))
- (testing "the tool refuses the query before returning a link"
- (is (= {:class :recoverable
- :code :metabase.metabot.tools.recoverable.common/not-found
- :text (str "Query q-join was not found. It may not exist, or you may not have access to it.\n"
- "Call `search` to find the entity you want and use an id from the results.")}
- (mt/with-current-user (mt/user->id :rasta)
- (failure {:query_id "q-join"}))))))))))))
+ (let [queries {"q-join" query}
+ path (mt/with-current-user (mt/user->id :crowberto)
+ (->> (invoke {:query_id "q-join"} queries)
+ :data-parts first :data :url))]
+ (testing "the X-ray API denies a query joining a card the user cannot read"
+ (is (= "You don't have permissions to do that."
+ (xray-page 403 path))))
+ (testing "the tool refuses the query before returning a link"
+ (is (= (failure ::recoverable.common/not-found
+ "Query q-join was not found. It may not exist, or you may not have access to it."
+ "Call `search` to find the entity you want and use an id from the results.")
+ (mt/with-current-user (mt/user->id :rasta)
+ (invoke {:query_id "q-join"} queries)))))))))))
(deftest draft-query-test
(testing "a draft query from the viewing context needs a data source for X-ray"
@@ -274,13 +240,11 @@
(let [context (api.macros/decode-and-validate-params
:body ::metabot.context/context
{:user_is_viewing [{:id "q-draft" :type "adhoc" :query draft}]})]
- (binding [shared/*memory-atom* (atom {:state (#'agent/seed-state {} context)})]
- (is (= {:class :recoverable
- :code ::autogen-dashboard/incomplete-query
- :text (str "Query q-draft has no data source yet, so X-ray cannot analyze it.\n"
- "Call `search` to find a table, model or metric to X-ray instead.")}
- (mt/with-current-user (mt/user->id :crowberto)
- (failure {:query_id "q-draft"}))))))))))
+ (is (= (failure ::autogen-dashboard/incomplete-query
+ "Query q-draft has no data source yet, so X-ray cannot analyze it."
+ "Call `search` to find a table, model or metric to X-ray instead.")
+ (mt/with-current-user (mt/user->id :crowberto)
+ (invoke {:query_id "q-draft"} (:queries (#'agent/seed-state {} context)))))))))))
(deftest sourceless-query-test
(testing "selecting a database without a table or card is not enough for an X-ray"
@@ -291,61 +255,37 @@
{:database (mt/id) :lib/type :mbql/query
:stages [{:lib/type :mbql.stage/mbql :source-table nil}]}]]
(testing (pr-str draft)
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-draft" draft}}})]
- (is (= {:class :recoverable
- :code ::autogen-dashboard/incomplete-query
- :text (str "Query q-draft has no data source yet, so X-ray cannot analyze it.\n"
- "Call `search` to find a table, model or metric to X-ray instead.")}
- (mt/with-current-user (mt/user->id :crowberto)
- (failure {:query_id "q-draft"})))))))))
+ (is (= (failure ::autogen-dashboard/incomplete-query
+ "Query q-draft has no data source yet, so X-ray cannot analyze it."
+ "Call `search` to find a table, model or metric to X-ray instead.")
+ (mt/with-current-user (mt/user->id :crowberto)
+ (invoke {:query_id "q-draft"} {"q-draft" draft}))))))))
(deftest unavailable-source-test
(mt/with-temp [:model/Card {model-id :id} {:type :model :archived true}
:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id) :active false}]
(mt/with-current-user (mt/user->id :rasta)
(testing "an archived source the user can read is in the trash"
- (is (= {:class :recoverable
- :code ::autogen-dashboard/unavailable-source
- :text (str "Model " model-id " is in the trash.\n"
- "Call `search` to find another source and use an id from the results.")}
- (failure {:model_id model-id}))))
+ (is (= (failure ::autogen-dashboard/unavailable-source
+ (str "Model " model-id " is in the trash.")
+ "Call `search` to find another source and use an id from the results.")
+ (invoke {:model_id model-id}))))
(testing "an inactive table is gone from its database"
- (is (= {:class :recoverable
- :code ::autogen-dashboard/unavailable-source
- :text (str "Table " table-id " is no longer in its database.\n"
- "Call `search` to find another source and use an id from the results.")}
- (failure {:table_id table-id})))))))
-
-(deftest one-source-test
- (testing "a call that sets two sources is rejected before either is read"
- (is (= {:class :recoverable
- :code ::autogen-dashboard/invalid-source
- :text "Set exactly one of `table_id`, `model_id`, `metric_id`, `query_id` or `report_id` in `source`."}
- (failure {:table_id 1 :model_id 2}))))
- (testing "a placeholder for an unused source is not a second source"
- (mt/with-temp [:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id)}
- :model/Card {model-id :id} {:type :model}]
- (mt/with-current-user (mt/user->id :rasta)
- (doseq [[source path] [[{:table_id nil :model_id model-id} (str "/auto/dashboard/model/" model-id)]
- [{:table_id table-id :model_id 0} (str "/auto/dashboard/table/" table-id)]
- [{:table_id 0 :model_id model-id} (str "/auto/dashboard/model/" model-id)]
- [{:query_id "" :table_id table-id} (str "/auto/dashboard/table/" table-id)]]]
- (testing (pr-str source)
- (is (=? {:structured-output {:path path}}
- (autogen-dashboard/create-autogenerated-dashboard-tool {:source source})))))))))
+ (is (= (failure ::autogen-dashboard/unavailable-source
+ (str "Table " table-id " is no longer in its database.")
+ "Call `search` to find another source and use an id from the results.")
+ (invoke {:table_id table-id})))))))
(deftest native-query-test
(testing "an unsaved SQL query gets a recoverable refusal instead of a broken X-ray link"
(doseq [query ((juxt identity lib/->legacy-MBQL)
(lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES"))]
(testing (pr-str (dissoc query :lib/metadata))
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-sql" query}}})]
- (is (= {:class :recoverable
- :code ::autogen-dashboard/unsupported-native-query
- :text (str "Query q-sql is an unsaved SQL query, which X-ray does not support.\n"
- "Call `search` to find a table, model, metric or saved question to X-ray instead.")}
- (mt/with-current-user (mt/user->id :rasta)
- (failure {:query_id "q-sql"})))))))))
+ (is (= (failure ::autogen-dashboard/unsupported-native-query
+ "Query q-sql is an unsaved SQL query, which X-ray does not support."
+ "Call `search` to find a table, model, metric or saved question to X-ray instead.")
+ (mt/with-current-user (mt/user->id :rasta)
+ (invoke {:query_id "q-sql"} {"q-sql" query}))))))))
(deftest saved-native-query-test
(testing "a saved SQL question has a working X-ray link"
@@ -354,18 +294,22 @@
"SELECT * FROM VENUES")})]
(mt/with-current-user (mt/user->id :rasta)
(is (=? {:dashcards seq}
- (->> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:report_id card-id}})
- :structured-output :path
+ (->> (invoke {:report_id card-id})
+ :data-parts first :data :url
(xray-page 200))))))))
(deftest unreadable-native-query-test
(testing "a SQL query that the user cannot run returns a not-found error"
- (binding [shared/*memory-atom* (atom {:state {:queries {"q-sql" (lib/native-query (mt/metadata-provider)
- "SELECT * FROM VENUES")}}})]
+ (let [queries {"q-sql" (lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES")}]
(mt/with-no-data-perms-for-all-users!
- (is (= {:class :recoverable
- :code :metabase.metabot.tools.recoverable.common/not-found
- :text (str "Query q-sql was not found. It may not exist, or you may not have access to it.\n"
- "Call `search` to find the entity you want and use an id from the results.")}
+ (is (= (failure ::recoverable.common/not-found
+ "Query q-sql was not found. It may not exist, or you may not have access to it."
+ "Call `search` to find the entity you want and use an id from the results.")
(mt/with-current-user (mt/user->id :rasta)
- (failure {:query_id "q-sql"}))))))))
+ (invoke {:query_id "q-sql"} queries))))))))
+
+(deftest unexpected-error-test
+ (testing "an error that no one declared ends the turn"
+ (mt/with-dynamic-fn-redefs [metabot.db/table (fn [_] (throw (ex-info "boom" {})))]
+ (is (= {:class :unrecoverable :code :internal}
+ (:error (invoke {:table_id 1})))))))
diff --git a/test/metabase/metabot/tools/entity_retrieval_test.clj b/test/metabase/metabot/tools/entity_retrieval_test.clj
index 2ad6fc91d9f..921e02f7e2e 100644
--- a/test/metabase/metabot/tools/entity_retrieval_test.clj
+++ b/test/metabase/metabot/tools/entity_retrieval_test.clj
@@ -2,6 +2,10 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.api-scope.core :as api-scope]
+ [metabase.metabot.scope :as scope]
+ [metabase.metabot.tools :as metabot.tools]
+ [metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.entity-retrieval :as entity-retrieval]
[metabase.metabot.tools.search :as tools.search]
[metabase.search.entity-retrieval.core :as cs.core]
@@ -18,6 +22,19 @@
`(mt/with-dynamic-fn-redefs [cs.core/search-unfiltered ~search-fn]
~@body))
+(defn- retrieve
+ "Call `retrieve_library_entities` with `args`."
+ [args]
+ (tools.core/handle entity-retrieval/retrieve-library-entities-tool args {}))
+
+(defn- invoke
+ "Call `retrieve_library_entities` with `args` the way the agent loop does. Return the result, or
+ `{:output <the text the model reads> :error {:class :code}}` when the call fails."
+ [args]
+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]
+ (metabot.tools/->entries [#'metabot.tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]
+ ((:fn (get entries "retrieve_library_entities")) args)))
+
(deftest tool-shape-oss-fallback-test
(testing "without the library-retrieval feature the tool returns the standard empty result shape"
;; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient
@@ -28,8 +45,7 @@
:data []
:total_count 0
:weak_match false}}
- (entity-retrieval/retrieve-library-entities-tool
- {:user_search_prompt "revenue per region"}))))))
+ (retrieve {:user_search_prompt "revenue per region"}))))))
(deftest similarity-helper-test
(let [similarity (var-get #'entity-retrieval/similarity)]
@@ -75,8 +91,7 @@
:doc_text sentinel
:score {:total_score 0.9
:scores [{:name :similarity :score 0.9}]}}]
- search! #(entity-retrieval/retrieve-library-entities-tool
- {:user_search_prompt "secret"})]
+ search! #(retrieve {:user_search_prompt "secret"})]
(with-unfiltered-search (fn [_ _] raw)
(mt/with-dynamic-fn-redefs
[cs.core/library-entity-keys (constantly #{["model" secret-id]})]
@@ -199,5 +214,13 @@
(testing "a failing library search propagates to the agent loop, which reports the call as failed"
(with-unfiltered-search (fn [_ _] (throw (ex-info "Embedding service unavailable" {})))
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Embedding service unavailable"
- (entity-retrieval/retrieve-library-entities-tool
- {:user_search_prompt "revenue per region"}))))))
+ (retrieve {:user_search_prompt "revenue per region"}))))))
+
+(deftest ^:parallel blank-search-prompt-test
+ (testing "a blank prompt goes back to the model before anything is searched"
+ (with-unfiltered-search (fn [_ _] (throw (ex-info "searched" {})))
+ (doseq [prompt ["" " "]]
+ (is (= {:output (str "`user_search_prompt` is blank. Set it to a full sentence that describes the data the "
+ "user wants.")
+ :error {:class :recoverable :code ::entity-retrieval/blank-search-prompt}}
+ (invoke {:user_search_prompt prompt})))))))
diff --git a/test/metabase/metabot/tools/run_query_test.clj b/test/metabase/metabot/tools/run_query_test.clj
index 334d0de648d..ed704d8ccc7 100644
--- a/test/metabase/metabot/tools/run_query_test.clj
+++ b/test/metabase/metabot/tools/run_query_test.clj
@@ -11,6 +11,7 @@
[metabase.metabot.scope :as scope]
[metabase.metabot.tools :as tools]
[metabase.metabot.tools.core :as tools.core]
+ [metabase.metabot.tools.recoverable.common :as recoverable.common]
[metabase.metabot.tools.run-query :as run-query]
[metabase.metabot.tools.sql.common :as sql.common]
[metabase.permissions.core :as perms]
@@ -170,9 +171,9 @@
(is (= (refusal ::run-query/query-execution-off "Query execution is turned off for Metabot.")
(call-tool! {:state {:queries {"q1" (venues-by-id)}}} {:query_id "q1"})))))
(testing "an unknown id lists the ids the model can use"
- (is (= (refusal ::run-query/unknown-query-id
- (str "No query with ID nope. Known query IDs: [q1]. Use the ID of a query that you built, or a "
- "saved question or model that the user can read."))
+ (is (= (refusal ::recoverable.common/unknown-query-id
+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". Use the ID of a query that you "
+ "built, or a saved question or model that the user can read."))
(run-tool! {"q1" (venues-by-id)} {:query_id "nope"}))))
(testing "a SQL query is refused with a pointer to construct_notebook_query while SQL execution is off"
(mt/with-temporary-setting-values [metabot-sql-execution-enabled? false]
@@ -522,8 +523,8 @@
(data-lines (:output (run-viewing! [] (str notebook-card)))))))
(testing "an id with anything but digits names no saved question"
(doseq [id [(str "+" notebook-card) (str notebook-card " ") (str "0" notebook-card)]]
- (is (=? {:output #"No query with ID .*\. Known query IDs: \[\]\. .*"
- :error {:code ::run-query/unknown-query-id}}
+ (is (=? {:output #"Query \".*\" was not found\. Available query IDs: none\. .*"
+ :error {:code ::recoverable.common/unknown-query-id}}
(run-viewing! [] id)))))
(testing "a viewed question the user can't read is refused like an id that names nothing"
(mt/with-non-admin-groups-no-root-collection-perms
@@ -865,8 +866,8 @@
(call-tool-in-conversation! {:state {:queries {}}
:context {:user_is_viewing viewing}}
{:query_id query-id})))
- unknown-id {:output #"No query with ID -?\d+\. Known query IDs: \[\]\. .*"
- :error {:code ::run-query/unknown-query-id}}
+ unknown-id {:output #"Query \"-?\d+\" was not found\. Available query IDs: none\. .*"
+ :error {:code ::recoverable.common/unknown-query-id}}
metric-refusal (fn [card-id step]
(refusal ::run-query/metric-not-runnable
(str "Query ID " card-id " is a metric, and a metric does not run by its ID.")
@@ -969,8 +970,9 @@
(deftest run-sql-query-tool-refusals-test
(testing "the SQL-only tool's refusals point at SQL or at telling the user, never at the notebook builder"
(testing "an unknown id"
- (is (= (refusal ::run-query/unknown-query-id
- "No query with ID nope. Known query IDs: [q1]. A saved question or model has no query ID."
+ (is (= (refusal ::recoverable.common/unknown-query-id
+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". A saved question or model has "
+ "no query ID.")
(str "To run one, write SQL that reads it with `create_sql_query`, then run that query with "
"`run_query`."))
(run-sql-only-tool! {"q1" (venues-by-id)} {:query_id "nope"}))))
@@ -999,9 +1001,9 @@
(is (= (refusal ::run-query/sql-not-allowed "This query is SQL, and SQL can't run here." no-values-step)
(run-sql-only-tool! false {"q1" (mt/native-query {:query "SELECT 1"})} {:query_id "q1"}))))
(testing "an unknown id"
- (is (= (refusal ::run-query/unknown-query-id
- (str "No query with ID nope. Known query IDs: [q1]. A saved question or model has no query ID, "
- "and no SQL can run here.")
+ (is (= (refusal ::recoverable.common/unknown-query-id
+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". A saved question or model has "
+ "no query ID, and no SQL can run here.")
no-values-step)
(run-sql-only-tool! false {"q1" (venues-by-id)} {:query_id "nope"}))))
(testing "a query that is not SQL"
@@ -1014,26 +1016,34 @@
(is (=? {:structured-output {:query-id "q1" :returned 1}}
(run-sql-only-tool! {"q1" (mt/native-query {:query "SELECT COUNT(*) FROM VENUES"})} {:query_id "q1"})))))
-(deftest run-sql-query-tool-many-query-ids-test
- (testing "a long list of known IDs is cut to leave room for the recovery step"
- (let [ids (repeatedly 200 u/generate-nano-id)
- {:keys [output error]} (run-sql-only-tool! (zipmap ids (repeat (venues-by-id))) {:query_id "nope"})
- [message step] (str/split-lines output)
- listed (some-> (re-find #"Known query IDs: \[([^\]]*)\] and 100 more\." message)
- second
- (str/split #", "))]
- (is (= ::run-query/unknown-query-id (:code error)))
- (is (= 100 (count listed)))
- (is (every? (set ids) listed))
- (is (= "To run one, write SQL that reads it with `create_sql_query`, then run that query with `run_query`."
- step)))))
+(deftest run-query-many-query-ids-test
+ (let [built-or-saved (str "Use the ID of a query that you built, or a saved question or model that the user can "
+ "read.")
+ write-sql (str "To run one, write SQL that reads it with `create_sql_query`, then run that query with "
+ "`run_query`.")]
+ (testing "a long list of known IDs is cut to complete IDs, so the advice after it survives the runtime's cut"
+ (doseq [[id-kind ids] {"NanoIDs" (repeatedly 200 u/generate-nano-id)
+ "UUIDs" (repeatedly 100 #(str (random-uuid)))}
+ [profile run advice] [["ordinary" run-tool! built-or-saved]
+ ["SQL-only" run-sql-only-tool! write-sql]]]
+ (testing (str profile " run_query, " id-kind)
+ (let [{:keys [output error]} (run (zipmap ids (repeat (venues-by-id))) {:query_id "nope"})
+ [_ shown more] (re-find #"Available query IDs: ([^.]*?)(?: and (\d+) more)?\." output)
+ listed (some->> shown (re-seq #"\"([^\"]*)\"") (map second))]
+ (is (= ::recoverable.common/unknown-query-id (:code error)))
+ (is (str/includes? output advice))
+ (is (every? (set ids) listed))
+ (is (= (- (count ids) (count listed)) (some-> more parse-long)))))))
+ (testing "an ID too long to fit is counted rather than cut"
+ (is (str/includes? (:output (run-tool! {(apply str (repeat 5000 "y")) (venues-by-id)} {:query_id "nope"}))
+ (str "Available query IDs: 1, too long to list. " built-or-saved))))))
(deftest run-sql-query-tool-huge-query-id-test
(testing "an invented ID is cut to leave room for the recovery step"
(let [query-id (apply str (repeat 5000 "x"))]
- (is (= (refusal ::run-query/unknown-query-id
- (str "No query with ID " (subs query-id 0 100) ".... Known query IDs: [q1]. A saved question or "
- "model has no query ID.")
+ (is (= (refusal ::recoverable.common/unknown-query-id
+ (str "Query \"" (subs query-id 0 100) "\" was not found. Available query IDs: \"q1\". A saved "
+ "question or model has no query ID.")
(str "To run one, write SQL that reads it with `create_sql_query`, then run that query with "
"`run_query`."))
(run-sql-only-tool! {"q1" (venues-by-id)} {:query_id query-id}))))))
@@ -1078,8 +1088,8 @@
metabot-sql-execution-enabled? true]
(binding [scope/*current-user-metabot-permissions* perms
scope/*current-user-scope* (scope/user-metabot-perms->scopes perms)]
- (is (=? {:output #"(?s)No query with ID \d+\. .*write SQL that reads it with `create_sql_query`.*"
- :error {:code ::run-query/unknown-query-id}}
+ (is (=? {:output #"(?s)Query \"\d+\" was not found\. .*write SQL that reads it with `create_sql_query`.*"
+ :error {:code ::recoverable.common/unknown-query-id}}
(call-sql-tool-in-conversation! {:state {:queries {}}
:context {:user_is_viewing [{:type "question", :id card-id}]}}
{:query_id (str card-id)})))))))))
diff --git a/test/metabase/metabot/tools/skills_test.clj b/test/metabase/metabot/tools/skills_test.clj
index daf52961ac2..ed772a20389 100644
--- a/test/metabase/metabot/tools/skills_test.clj
+++ b/test/metabase/metabot/tools/skills_test.clj
@@ -2,40 +2,106 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
+ [metabase.metabot.scope :as scope]
[metabase.metabot.skills :as skills]
+ [metabase.metabot.tools :as metabot.tools]
+ [metabase.metabot.tools.core :as tools.core]
+ [metabase.metabot.tools.error :as tools.error]
+ [metabase.metabot.tools.runtime :as tools.runtime]
[metabase.metabot.tools.skills :as tools.skills]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.log.capture :as log.capture]
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
-(deftest ^:parallel load-skill-tool-metadata-test
- (testing "the tool var carries the metadata profiles require"
- (let [m (meta #'tools.skills/load-skill-tool)]
- (is (= "load_skill" (:tool-name m)))
- (is (some? (:schema m))))))
+(defn- load-skills
+ "Call `load_skill` for `ids` the way the agent loop does, in a request whose skill manifest holds `loadable`.
+ Return the result, or `{:output <the text the model reads> :error {:class :code}}` when the call fails."
+ [loadable ids]
+ (binding [scope/*current-loadable-skill-ids* (atom (set loadable))]
+ (let [entries (metabot.tools/->entries [#'metabot.tools/load-skill-tool] (atom {}) nil :internal)]
+ ((:fn (get entries "load_skill")) {:ids ids}))))
+
+(deftest ^:parallel load-skill-takes-ids-test
+ (testing "the model sends `ids`, and so does the synthetic call that preloads a SQL dialect"
+ (let [{:keys [args]} (tools.core/validate-tool! tools.skills/load-skill-tool)
+ [preload] (skills/dialect-preload-parts "postgresql")]
+ (is (= "load_skill" (:function preload)))
+ (is (mr/validate args (:arguments preload))))))
(deftest ^:parallel load-skill-returns-body-test
(testing "loading a known skill returns its body wrapped in a <skill> tag"
- (let [out (:output (tools.skills/load-skill-tool {:ids ["construct-notebook-query-operators"]}))]
- (is (str/includes? out "<skill id=\"construct-notebook-query-operators\">"))
- (is (str/includes? out "</skill>"))
- (is (str/includes? out "Aggregations")))))
+ (let [{:keys [output error]} (load-skills [:construct-notebook-query-operators]
+ ["construct-notebook-query-operators"])]
+ (is (nil? error))
+ (is (str/starts-with? output "<skill id=\"construct-notebook-query-operators\">\n"))
+ (is (str/ends-with? output "\n</skill>"))
+ (is (str/includes? output "Aggregations")))))
(deftest ^:parallel load-skill-multiple-test
- (testing "multiple ids are concatenated"
- (let [out (:output (tools.skills/load-skill-tool
- {:ids ["construct-notebook-query-core" "read-resource"]}))]
- (is (str/includes? out "<skill id=\"construct-notebook-query-core\">"))
- (is (str/includes? out "<skill id=\"read-resource\">")))))
+ (testing "several skills come back in the order asked for, separated by a blank line"
+ (let [{:keys [output]} (load-skills [:construct-notebook-query-core :read-resource]
+ ["read-resource" "construct-notebook-query-core"])]
+ (is (= ["read-resource" "construct-notebook-query-core"]
+ (map second (re-seq #"<skill id=\"([^\"]+)\">" output))))
+ (is (str/includes? output "</skill>\n\n<skill id=\"construct-notebook-query-core\">")))))
(deftest ^:parallel load-skill-unknown-test
- (testing "an unknown id yields an inline message, not an exception"
- (let [out (:output (tools.skills/load-skill-tool {:ids ["does-not-exist"]}))]
- (is (str/includes? out "Unknown skill"))
- (is (str/includes? out "does-not-exist")))))
+ (testing "a skill that doesn't exist and one that this request may not load fail the same way"
+ (doseq [id ["does-not-exist" "edit-chart"]]
+ (testing id
+ (is (= {:output (str "No skill with ID \"" id "\" can be loaded here. Skill IDs you can load: "
+ "construct-notebook-query-core, read-resource.")
+ :error {:class :recoverable :code ::tools.core/all-items-failed}}
+ (load-skills [:read-resource :construct-notebook-query-core] [id]))))))
+ (testing "each skill that doesn't load is the declared error"
+ (binding [scope/*current-loadable-skill-ids* (atom #{})]
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (tools.core/handle tools.skills/load-skill-tool {:id "read-resource"} {})))]
+ (is (= ::tools.skills/unknown-skill (:code (tools.error/classify e)))))))
+ (testing "when the request may load no skills, the model is told so"
+ (is (= "No skill with ID \"read-resource\" can be loaded here. No skills can be loaded here."
+ (:output (load-skills [] ["read-resource"])))))
+ (testing "a long ID is cut short, so the list of skills stays within the failure text limit"
+ (is (str/ends-with? (:output (load-skills [:read-resource] [(apply str (repeat 5000 "x"))]))
+ "...\" can be loaded here. Skill IDs you can load: read-resource.")))
+ (testing "the skills that load still reach the model, and the failure keeps its place"
+ (let [{:keys [output error]} (load-skills [:read-resource] ["read-resource" "does-not-exist"])]
+ (is (nil? error))
+ (is (str/starts-with? output "<skill id=\"read-resource\">"))
+ (is (str/ends-with? output (str "</skill>\n\nNo skill with ID \"does-not-exist\" can be loaded here. "
+ "Skill IDs you can load: read-resource."))))))
+
+(deftest ^:parallel load-skill-lists-loadable-skills-once-test
+ (testing "several IDs that don't load list the skills that can be loaded once, after the first of them"
+ (is (= {:output (str "No skill with ID \"nope-1\" can be loaded here. Skill IDs you can load: read-resource.\n\n"
+ "No skill with ID \"nope-2\" can be loaded here.")
+ :error {:class :recoverable :code ::tools.core/all-items-failed}}
+ (load-skills [:read-resource] ["nope-1" "nope-2"]))))
+ (testing "a call whose failures run past the failure text limit is cut short, and still lists the skills"
+ (let [ids (for [i (range 40)] (str i (apply str (repeat 200 "x"))))
+ {:keys [output error]} (load-skills [:read-resource] ids)]
+ (is (= {:class :recoverable :code ::tools.core/all-items-failed} error))
+ (is (= tools.runtime/max-output-length (count output)))
+ (is (= 1 (count (re-seq #"Skill IDs you can load: read-resource\." output)))))))
+
+(deftest ^:parallel load-skill-logs-refused-ids-test
+ (testing "a missing or hidden skill leaves an info line, so operators can see a model guessing skill IDs"
+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.skills :info]]
+ (load-skills [:read-resource] ["does-not-exist" "edit-chart" "read-resource"])
+ (is (= ["Unknown skill requested by load_skill"
+ "Skill not available to current request {:id edit-chart}"]
+ (map :message (logs)))))))
+
+(deftest ^:parallel load-skill-needs-an-id-test
+ (testing "an empty `ids` is an argument error that the model can correct"
+ (is (= {:output "Invalid tool arguments: `ids` must be an array of at least one skill id; received an array."
+ :error {:class :validation :code :invalid-arguments}}
+ (load-skills [:read-resource] [])))))
(deftest ^:parallel load-skill-unexpected-error-test
- (testing "an unexpected error propagates to the agent loop"
+ (testing "an unexpected error fails the whole call"
(mt/with-dynamic-fn-redefs [skills/get-skill-by-id-string (fn [_] (throw (ex-info "boom" {})))]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
- (tools.skills/load-skill-tool {:ids ["construct-notebook-query-core"]}))))))
+ (is (= {:class :unrecoverable :code :internal}
+ (:error (load-skills [:construct-notebook-query-core] ["construct-notebook-query-core"])))))))
diff --git a/test/metabase/metabot/tools/timelines_test.clj b/test/metabase/metabot/tools/timelines_test.clj
index 57dbbbf2f31..1b3011a09c0 100644
--- a/test/metabase/metabot/tools/timelines_test.clj
+++ b/test/metabase/metabot/tools/timelines_test.clj
@@ -8,30 +8,28 @@
[metabase.metabot.tools.timelines :as tools.timelines]
[metabase.test :as mt]))
+(defn- list-timelines
+ "Call the `list_timelines` tool and return what the model reads."
+ []
+ (:output (tools.core/handle tools.timelines/list-timelines-tool {} {})))
+
(deftest list-timelines-tool-test
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Collection {coll-id :id} {:name "Timeline Coll"}
:model/Timeline {tl-id :id} {:name "Product Launches"
:description "Key product launch dates"
:collection_id coll-id}]
- (testing "returns timelines with id, name, description"
- (let [result (tools.timelines/list-timelines-tool {})
- items (:structured_output result)
- matching (filter #(= tl-id (:id %)) items)]
- (is (seq matching) "expected the test timeline to appear")
- (is (= {:id tl-id :name "Product Launches" :description "Key product launch dates"}
- (first matching)))))
- (testing "output contains XML-formatted text"
- (let [{:keys [output]} (tools.timelines/list-timelines-tool {})]
- (is (re-find #"<timelines>" output))
- (is (re-find #"Product Launches" output))))
+ (testing "each timeline reaches the model as XML, with its id, name and description"
+ (let [output (list-timelines)]
+ (is (str/starts-with? output "<timelines>\n"))
+ (is (str/includes? output (str "<timeline id=\"" tl-id "\" name=\"Product Launches\">"
+ "Key product launch dates</timeline>")))
+ (is (str/ends-with? output "\n</timelines>"))))
(testing "archived timelines are excluded by default"
(mt/with-temp [:model/Timeline _ {:name "Archived TL"
:collection_id coll-id
:archived true}]
- (let [items (:structured_output (tools.timelines/list-timelines-tool {}))
- names (set (map :name items))]
- (is (not (contains? names "Archived TL")))))))))
+ (is (not (str/includes? (list-timelines) "Archived TL"))))))))
(defn- details
"Call the converted `get_timeline_details` tool for `timeline-id`."
@@ -92,7 +90,7 @@
:timestamp #t "2025-06-01T00:00:00Z"
:icon "star"}]
(testing "list output escapes XML-special characters in names and descriptions"
- (let [{:keys [output]} (tools.timelines/list-timelines-tool {})]
+ (let [output (list-timelines)]
(is (re-find #"name=\"Q&amp;A &lt;&quot;Launches&quot;&gt;\"" output))
(is (re-find #"Tags: &lt;b&gt; &amp; &quot;quotes&quot;" output))))
(testing "details output escapes XML-special characters in timeline and event fields"
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 6da26a4a12a..55429548045 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -24,6 +24,7 @@
[metabase.search.entity-retrieval.core :as entity-retrieval]
[metabase.test :as mt]
[metabase.util.json :as json]
+ [metabase.util.malli :as mu]
[toucan2.core :as t2]))
(deftest all-tools-test
@@ -886,6 +887,13 @@
{:query {:lib/type "mbql/query" :stages [stage]}
:title "Orders" :description "Orders by count."})))))))))
+(mu/defn ^{:tool-name "unconverted"}
+ unconverted-tool
+ "A tool declared through its var's metadata. The real ones are converted one at a time, so the tests
+ keep their own."
+ [{:keys [text]} :- [:map {:closed true} [:text :string]]]
+ text)
+
(deftest ->entries-test
(testing "an entry carries what the adapters and the system message read"
(let [entries (agent-tools/->entries [#'agent-tools/create-sql-query-tool
@@ -900,15 +908,15 @@
(is (some? (:schema entry)))
(is (fn? (:fn entry))))))))
(testing "an unconverted tool's schema comes from its var metadata"
- (let [entries (agent-tools/->entries [#'agent-tools/list-timelines-tool] (atom {}) nil :nlq)
- [_:=> [_:cat declared] _out] (:schema (get entries "list_timelines"))
- [_:=> [_:cat original] _out] (:schema (meta #'agent-tools/list-timelines-tool))]
+ (let [entries (agent-tools/->entries [#'unconverted-tool] (atom {}) nil :nlq)
+ [_:=> [_:cat declared] _out] (:schema (get entries "unconverted"))
+ [_:=> [_:cat original] _out] (:schema (meta #'unconverted-tool))]
(is (= original declared))))
(testing "a converted tool and an unconverted one sit side by side"
- (let [entries (agent-tools/->entries [#'agent-tools/list-timelines-tool ; mu/defn var
+ (let [entries (agent-tools/->entries [#'unconverted-tool ; mu/defn var
#'agent-tools/get-timeline-details-tool] ; record
(atom {}) nil :nlq)]
- (is (= #{"list_timelines" "get_timeline_details"} (set (keys entries))))
+ (is (= #{"unconverted" "get_timeline_details"} (set (keys entries))))
(is (every? fn? (map :fn (vals entries))))))
(testing "the entry's :fn goes through the runtime, so it reports rather than throws"
(let [entries (agent-tools/->entries [#'agent-tools/search-tool] (atom {}) nil :nlq)