The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Convert four more Metabot tools to the tool contract (#85037)
Andrei Simionescue66356efd549
diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.cljindex 3c9f6248183..95212c514b7 100644--- a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj@@ -8,6 +8,7 @@[metabase-enterprise.search.semantic.embedding :as semantic.embedding][metabase-enterprise.search.semantic.embedding-health :as embedding-health][metabase-enterprise.search.semantic.test-util :as semantic.tu]+ [metabase.metabot.tools.core :as tools.core][metabase.metabot.tools.entity-retrieval :as tools.entity-retrieval][metabase.search.entity-retrieval.mirror :as mirror][metabase.test :as mt]@@ -434,7 +435,9 @@;; bind a user: the tool's hydration permission-filters via mi/can-read?, which needs;; *current-user-id* (a superuser can read the test tables).search! #(mt/with-test-user :crowberto- (get-in (tools.entity-retrieval/retrieve-library-entities-tool {:user_search_prompt q})+ (get-in (tools.core/handle tools.entity-retrieval/retrieve-library-entities-tool+ {:user_search_prompt q}+ {})[:structured-output :data]))](mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model(constantly semantic.tu/mock-embedding-model)]diff --git a/src/metabase/metabot/tools/autogen_dashboard.clj b/src/metabase/metabot/tools/autogen_dashboard.cljindex b976fc3a85d..bcaeaa705d9 100644--- a/src/metabase/metabot/tools/autogen_dashboard.clj+++ b/src/metabase/metabot/tools/autogen_dashboard.clj@@ -1,5 +1,5 @@(ns metabase.metabot.tools.autogen-dashboard- "Autogenerated dashboard tool wrappers."+ "The `create_autogenerated_dashboard` tool."(:require[clojure.string :as str][medley.core :as m]@@ -10,9 +10,9 @@[metabase.metabot.db :as metabot.db][metabase.metabot.scope :as scope][metabase.metabot.tools.core :as tools]+ [metabase.metabot.tools.recoverable.common :as recoverable.common][metabase.metabot.tools.shared :as shared][metabase.util.log :as log]- [metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.xrays.core :as xrays]))@@ -79,10 +79,26 @@(or (pos-int? v)(and (string? v) (not (str/blank? v)))))+(defn- normalized-query+ "`query`, one of the conversation's queries, normalized, or nil when it names no database or query type yet."+ [query]+ (when (and (:database query) (lib/normalized-query-type query))+ (lib-be/normalize-query query)))++(defn- analyzable-query-ids+ "The IDs of the conversation's `queries` that X-ray can analyze, sorted: the complete queries that don't start+ with SQL."+ [queries]+ (sort (for [[id query] queries+ :let [mbql-query (normalized-query query)]+ :when (and (mr/validate :metabase.lib.schema/query mbql-query)+ (not (lib/native? mbql-query)))]+ id)))+(defn- resolve-source- "Resolve the source entity from the provided source parameters.- Returns the entity to analyze and the path for navigation."- [source memory]+ "The X-ray path for `source` under `:path`, and the table, card or query that it reads under `:entity`.+ A `query_id` names one of `queries`, the conversation's queries."+ [source queries](let [{:keys [table_id model_id metric_id query_id report_id] :as source} (m/filter-vals source-value? source)](when-not (= 1 (count source))(invalid-source! {}))@@ -93,16 +109,12 @@report_id (readable-source :question report_id)query_id- (let [queries-state (get-in memory [:state :queries] {})- query (get queries-state query_id)- mbql-query (when (and (:database query) (lib/normalized-query-type query))- (lib-be/normalize-query query))]+ (let [query (get queries query_id)+ mbql-query (normalized-query query)](when-not query- (throw (ex-info (str "Query not found in memory with ID: " query_id- ". Available queries: [" (str/join ", " (keys queries-state)) "]")- {:agent-error? true- :query-id query_id- :available-queries (keys queries-state)})))+ (recoverable.common/unknown-query-id! {:query-id query_id+ :available (analyzable-query-ids queries)+ :hint :notebook-only}))(when-not (mr/validate :metabase.lib.schema/query mbql-query)(incomplete-query! {:query-id query_id}))(let [entity (tools/with-entity {:kind :query :id query_id}@@ -112,55 +124,6 @@{:entity entity:path (str "/auto/dashboard/adhoc/" (xrays/encode-base64-json query))})))))-;;; Response Messages--(def ^:private response-messages- ["Ok, here is a dashboard that may answer it for you."- "I've created a dashboard that may answer your question."- "Here is a dashboard that may help you with your question."- "I've generated a dashboard with insights about your data."])--(defn- random-response-message []- (rand-nth response-messages))--;;; Main Tool Implementation--(defn create-autogenerated-dashboard- "Create an auto-generated dashboard using Metabase's X-ray feature.-- X-ray builds complete dashboards with multiple visualizations that showcase- different aspects of your data - distributions, breakdowns by categories,- trends over time, and relationships between fields.-- Parameters:- - source: Map specifying what to analyze. One of:- - {:table_id 123} - Create dashboard for a table- - {:model_id 456} - Create dashboard for a model- - {:metric_id 789} - Create dashboard for a metric- - {:query_id \"abc\"} - Create dashboard for notebook query results (SQL queries must be saved first)- - {:report_id 101} - Create dashboard for an existing report/question- - memory-atom: Atom containing agent memory (for query resolution)-- Returns map with:- - :structured-output - Message and dashboard info- - :data-parts - Dashboard `generated_entity` part for the frontend- - :instructions - Guidance for the LLM on how to respond"- [{:keys [source memory-atom]}]- (log/info "Creating autogenerated dashboard" {:source source})-- (let [memory (when memory-atom @memory-atom)- {:keys [entity path]} (resolve-source source memory)- title (or (:display_name entity) (:name entity) "Dashboard")- response-message (str (random-response-message) "\n\nLet me know if you need anything else.")]- ;; Note: The actual automagic analysis happens on navigation to the /auto/dashboard/... URL- ;; The frontend handles rendering the X-ray dashboard-- (log/info "Autogenerated dashboard created" {:source source})- {:structured-output {:message response-message- :path path}- :data-parts [(streaming/dashboard-entity-part {:title title :url path})]- :instructions "The dashboard has been created and the user has been navigated to view it. Briefly acknowledge this (1 sentence) and stop. Do not describe the dashboard contents."}))-(def ^:private autogen-dashboard-schema[:map {:closed true}[:source [:map {:closed true}@@ -170,28 +133,34 @@[:query_id {:optional true} [:maybe :string]][:report_id {:optional true} [:maybe :int]]]]])-(mu/defn ^{:tool-name "create_autogenerated_dashboard"- :scope scope/agent-dashboard-create}- create-autogenerated-dashboard-tool- "Create an automatically generated dashboard using Metabase's X-ray feature.-- X-ray builds complete dashboards with multiple visualizations that showcase- different aspects of your data - distributions, breakdowns by categories,- trends over time, and relationships between fields.-- Specify one source type:- - table_id: Create dashboard for a table- - model_id: Create dashboard for a model- - metric_id: Create dashboard for a metric- - query_id: Create dashboard for notebook queries from this conversation (SQL queries must be saved first)- - report_id: Create dashboard for an existing report/question"- [{:keys [source]} :- autogen-dashboard-schema]- (let [result (create-autogenerated-dashboard- {:source source- :memory-atom shared/*memory-atom*})- structured (:structured-output result)- data-parts (:data-parts result)]- (cond-> {:output (:message structured)- :structured-output structured- :instructions (:instructions result)}- (seq data-parts) (assoc :data-parts data-parts))))+(defrecord CreateAutogeneratedDashboardTool []+ tools/Tool+ (declaration [_]+ {:name "create_autogenerated_dashboard"+ :description (str "Create an automatically generated dashboard using Metabase's X-ray feature.\n\n"+ "X-ray builds complete dashboards with multiple visualizations that showcase different "+ "aspects of your data - distributions, breakdowns by categories, trends over time, and "+ "relationships between fields.\n\n"+ "Specify one source type:\n"+ "- table_id: Create dashboard for a table\n"+ "- model_id: Create dashboard for a model\n"+ "- metric_id: Create dashboard for a metric\n"+ "- query_id: Create dashboard for notebook queries from this conversation (SQL queries must "+ "be saved first)\n"+ "- report_id: Create dashboard for an existing report/question")+ :scope scope/agent-dashboard-create+ :args autogen-dashboard-schema})++ (handle [_ {:keys [source]} {:keys [memory-atom]}]+ (log/info "Creating autogenerated dashboard" {:source source})+ ;; The X-ray itself runs when the frontend opens `path`+ (let [{:keys [entity path]} (resolve-source source (shared/current-queries-state memory-atom))]+ (log/info "Autogenerated dashboard created" {:source source})+ {:output (str "The dashboard has been created and the user has been navigated to view it. Briefly "+ "acknowledge this (1 sentence) and stop. Do not describe the dashboard contents.")+ :data-parts [(streaming/dashboard-entity-part {:title (or (:display_name entity) (:name entity) "Dashboard")+ :url path})]})))++(def create-autogenerated-dashboard-tool+ "The `create_autogenerated_dashboard` tool."+ (->CreateAutogeneratedDashboardTool))diff --git a/src/metabase/metabot/tools/entity_retrieval.clj b/src/metabase/metabot/tools/entity_retrieval.cljindex c38be67c381..312bea61706 100644--- a/src/metabase/metabot/tools/entity_retrieval.clj+++ b/src/metabase/metabot/tools/entity_retrieval.clj@@ -17,11 +17,11 @@(:require[clojure.string :as str][metabase.metabot.scope :as scope]+ [metabase.metabot.tools.core :as tools][metabase.metabot.tools.search :as tools.search][metabase.metabot.tools.shared.llm-shape :as llm-shape][metabase.search.entity-retrieval.core :as entity-retrieval]- [metabase.util :as u]- [metabase.util.malli :as mu]))+ [metabase.util :as u]))(set! *warn-on-reflection* true)@@ -167,29 +167,46 @@:confidence (if weak? "weak" "strong")))matches))+(tools/defrecoverable blank-search-prompt!+ "`user_search_prompt` is empty or only whitespace."+ {:payload [:map {:closed true}]}+ [_]+ {:message "`user_search_prompt` is blank. Set it to a full sentence that describes the data the user wants."+ :recovery []})+;; No capability gate: this tool lives only in the :nlq profile, which [[metabase.metabot.agent.profiles/;; get-profile]] serves (the library variant, with this tool) only when entity-retrieval-available? — otherwise it;; redirects to :nlq-fallback (general search). That single probe is the sole gate, so prompt and tools can't;; disagree about index availability.-(mu/defn ^{:tool-name "retrieve_library_entities"- :scope scope/agent-search}- retrieve-library-entities-tool- "Find the best data to answer the user's request from the library — the set of entities published to it- (published tables, library metrics/models, and their measures/segments). Phrase- `user_search_prompt` as a full natural-language description of the data wanted (it is matched on- meaning, not keywords).-- Returns a handful of distinct library entities, best-first. Each match has the `matched_on` text that- triggered it, any curator `usage_instructions`, a raw `similarity`, and the `entity` — a full search- record (name, type, database, `portable_entity_id`, fully-qualified name). Use it to pick the right- entity, then `read_resource` that entity to confirm its fields and sample values before building. If the- top match is flagged low-confidence (a leading <note> / confidence=\"weak\"), nothing in the library- clearly matches — prefer asking the user to clarify or narrow the request."- [{:keys [user_search_prompt limit]} :- entity-retrieval-schema]- (let [n (min max-limit (or limit default-limit))- matches (build-matches user_search_prompt n)]- {:output (format-output matches)- :structured-output {:result-type :search- :data (flatten-data matches)- :total_count (count matches)- :weak_match (boolean (:weak? (first matches)))}}))+(defrecord RetrieveLibraryEntitiesTool []+ tools/Tool+ (declaration [_]+ {:name "retrieve_library_entities"+ :description (str "Find the best data to answer the user's request from the library – the set of entities "+ "published to it (published tables, library metrics/models, and their measures/segments). "+ "Phrase `user_search_prompt` as a full natural-language description of the data wanted (it "+ "is matched on meaning, not keywords).\n\n"+ "Returns a handful of distinct library entities, best-first. Each match has the `matched_on` "+ "text that triggered it, any curator `usage_instructions`, a raw `similarity`, and the "+ "`entity` – a full search record (name, type, database, `portable_entity_id`, "+ "fully-qualified name). Use it to pick the right entity, then `read_resource` that entity to "+ "confirm its fields and sample values before building. If the top match is flagged "+ "low-confidence (a leading <note> / confidence=\"weak\"), nothing in the library clearly "+ "matches – prefer asking the user to clarify or narrow the request.")+ :scope scope/agent-search+ :args entity-retrieval-schema})++ (handle [_ {:keys [user_search_prompt limit]} _ctx]+ (when (str/blank? user_search_prompt)+ (blank-search-prompt! {}))+ (let [n (min max-limit (or limit default-limit))+ matches (build-matches user_search_prompt n)]+ {:output (format-output matches)+ :structured-output {:result-type :search+ :data (flatten-data matches)+ :total_count (count matches)+ :weak_match (boolean (:weak? (first matches)))}})))++(def retrieve-library-entities-tool+ "The `retrieve_library_entities` tool."+ (->RetrieveLibraryEntitiesTool))diff --git a/src/metabase/metabot/tools/error.clj b/src/metabase/metabot/tools/error.cljindex bf92129c55a..ca2a571da12 100644--- a/src/metabase/metabot/tools/error.clj+++ b/src/metabase/metabot/tools/error.clj@@ -346,6 +346,19 @@(cons message)(str/join "\n")))+(def max-output-length+ "Maximum number of characters in a rendered failure, including recovery steps.++ Leave room for recovery steps when including excerpts."+ 4000)++(defn output-budget+ "Return the character budget for an excerpt after `prefix` and every recovery step."+ [prefix recovery]+ (let [tool-names (into #{} (mapcat :uses) recovery)]+ (- max-output-length+ (count (recoverable-text {:message prefix :recovery recovery} tool-names)))))+(defn authored-text"Return `text` with the quoted payload values from `error`'s declaration removed.diff --git a/src/metabase/metabot/tools/recoverable/common.clj b/src/metabase/metabot/tools/recoverable/common.cljindex 2f4ce8a5847..924c45396ee 100644--- a/src/metabase/metabot/tools/recoverable/common.clj+++ b/src/metabase/metabot/tools/recoverable/common.clj@@ -6,7 +6,7 @@specific."(:require[clojure.string :as str]- [metabase.metabot.tools.error :refer [defrecoverable]]+ [metabase.metabot.tools.error :as tools.error :refer [defrecoverable]][metabase.util :as u][metabase.util.quoting :as quoting]))@@ -47,18 +47,76 @@[{:uses #{finder}:text (str "Call `" finder "` to find the entity you want and use an id from the results.")}])})+(def ^:private max-query-id-chars+ "The most characters of the requested query ID that [[unknown-query-id!]] repeats.+ The model supplies the ID, so its length is otherwise unbounded."+ 100)++(defn- available-query-ids-text+ "`ids` quoted and joined within `room` characters: the leading IDs that fit whole, then a count of the rest."+ [ids room]+ (let [quoted (mapv quoting/quoted ids)+ n (count quoted)+ running-lengths (reductions + (map-indexed (fn [i id] (+ (count id) (if (pos? i) 2 0))) quoted))]+ (cond+ (zero? n) "none"+ (<= (last running-lengths) room) (str/join ", " quoted)+ :else+ (let [limit (- room (count (str " and " n " more")))+ shown (count (take-while #(<= % limit) running-lengths))]+ (if (pos? shown)+ (str (str/join ", " (take shown quoted)) " and " (- n shown) " more")+ (str n ", too long to list"))))))++(defn- unknown-query-id-advice+ "What [[unknown-query-id!]] tells the model to do for `hint`: a `:sentence` after its message, and its `:recovery`."+ [hint any-available?]+ (let [use-available {:uses #{} :text "Use an available query ID from this conversation."}]+ (case hint+ nil+ {:recovery [(if any-available?+ use-available+ {:uses #{"create_sql_query"}+ :text "Use `create_sql_query` to create a SQL query, then use the returned query ID."})]}++ :notebook-only+ {:recovery [(if any-available?+ use-available+ {:uses #{"construct_notebook_query"}+ :text "Use `construct_notebook_query` to build a query, then use the returned query ID."})]}++ :notebook-query+ {:sentence " Use the ID of a query that you built, or a saved question or model that the user can read."+ :recovery []}++ :sql-query+ {:sentence " A saved question or model has no query ID."+ :recovery [{:uses #{"create_sql_query" "run_query"}+ :text (str "To run one, write SQL that reads it with `create_sql_query`, then run that query"+ " with `run_query`.")}]}++ :no-values+ {:sentence " A saved question or model has no query ID, and no SQL can run here."+ :recovery [{:uses #{} :text "Tell the user you can't read its results."}]})))+(defrecoverable unknown-query-id!- "The requested query is absent from the conversation."+ "The query ID names no query that the tool can use for the current user.++ `:hint` picks the recovery. By default the model uses a listed ID or creates a SQL query, and `:notebook-only`+ offers a notebook query instead. `run_query` passes one of the others for its profile, since it also runs saved+ questions by ID."{:payload [:map {:closed true}- [:query-id :string]- [:available [:sequential :string]]]}- [{:keys [query-id available]}]- {:message (str "Query " (quoting/quoted query-id) " was not found. Available query IDs: "- (if (seq available) (str/join ", " (map quoting/quoted available)) "none") ".")- :recovery [(if (seq available)- {:uses #{} :text "Use an available query ID from this conversation."}- {:uses #{"create_sql_query"}- :text "Use `create_sql_query` to create a SQL query, then use the returned query ID."})]})+ [:query-id :string]+ [:available [:sequential :string]]+ [:hint {:optional true} [:enum :notebook-only :notebook-query :sql-query :no-values]]]}+ [{:keys [query-id available hint]}]+ (let [{:keys [sentence recovery]} (unknown-query-id-advice hint (boolean (seq available)))+ opening (str "Query " (quoting/quoted (u/truncate query-id max-query-id-chars))+ " was not found. Available query IDs: ")+ ;; The IDs get only the room left by the rest, so a long list never pushes the advice past the runtime's cut+ room (tools.error/output-budget (str opening "." sentence) recovery)]+ {:message (str opening (available-query-ids-text available room) "." sentence)+ :recovery recovery}))(defrecoverable unknown-database!"The requested database does not exist."diff --git a/src/metabase/metabot/tools/run_query.clj b/src/metabase/metabot/tools/run_query.cljindex 438a9d10e61..ab77ee0008b 100644--- a/src/metabase/metabot/tools/run_query.clj+++ b/src/metabase/metabot/tools/run_query.clj@@ -14,6 +14,7 @@[metabase.metabot.settings :as metabot.settings][metabase.metabot.tmpl :as te][metabase.metabot.tools.core :as tools]+ [metabase.metabot.tools.recoverable.common :as recoverable.common][metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.llm-shape :as llm-shape][metabase.metabot.tools.sql.common :as sql.common]@@ -89,46 +90,6 @@:recovery [{:uses #{}:text "Ask the user to continue in their own Metabot chat."}]})-(def ^:private max-listed-query-ids- "The refusal lists at most this many known query IDs.-- This leaves room for a recovery step within the runtime's limit on failure text."- 100)--(def ^:private max-echoed-query-id-chars- "The refusal repeats at most this many characters from an unknown query ID.-- The model supplies the ID, so its length is otherwise unbounded."- 100)--(tools/defrecoverable unknown-query-id!- "The query ID matches no stored query or saved question that this tool can run for the current user."- {:payload [:map {:closed true}- [:query-id :string]- [:known-ids [:sequential :string]]- [:hint [:enum :notebook-query :sql-query :no-values]]]}- [{:keys [query-id known-ids hint]}]- (let [more (- (count known-ids) max-listed-query-ids)- unknown (str "No query with ID " (llm-shape/truncate query-id max-echoed-query-id-chars) ". Known query IDs: ["- (str/join ", " (take max-listed-query-ids known-ids)) "]"- (when (pos? more) (str " and " more " more"))- ".")]- (case hint- :notebook-query- {:message (str unknown " Use the ID of a query that you built, or a saved question or model that the user"- " can read.")- :recovery []}-- :sql-query- {:message (str unknown " A saved question or model has no query ID.")- :recovery [{:uses #{"create_sql_query" "run_query"}- :text (str "To run one, write SQL that reads it with `create_sql_query`, then run that query"- " with `run_query`.")}]}-- :no-values- {:message (str unknown " A saved question or model has no query ID, and no SQL can run here.")- :recovery [(:no-values hint-steps)]})))-(tools/defrecoverable metric-not-runnable!"The ID names a metric that the user can read, and a metric does not run by its ID.A metric is an aggregation, not a source, so no query with it as the source would run."@@ -292,16 +253,17 @@"Return the query named by `query-id` in a map under `:query`.Saved questions are available only when `allow-non-sql?` is true and the user can read them;- the result then includes the card under `:saved-card`. Throw [[unknown-query-id!]] if no query is found."+ the result then includes the card under `:saved-card`. Throw [[recoverable.common/unknown-query-id!]] if no query+ is found."[query-id allow-non-sql?](let [queries (shared/current-queries-state)](or (when-let [query (get queries query-id)]{:query query})(when allow-non-sql?(saved-question query-id))- (unknown-query-id! {:query-id query-id- :known-ids (vec (keys queries))- :hint (recovery-hint allow-non-sql?)}))))+ (recoverable.common/unknown-query-id! {:query-id query-id+ :available (sort (keys queries))+ :hint (recovery-hint allow-non-sql?)}))))(defn- check-private-conversation!"Refuse unless the current conversation belongs to the current user alone.diff --git a/src/metabase/metabot/tools/runtime.clj b/src/metabase/metabot/tools/runtime.cljindex 77cb3fdc10a..8e0eae2db11 100644--- a/src/metabase/metabot/tools/runtime.clj+++ b/src/metabase/metabot/tools/runtime.clj@@ -28,10 +28,17 @@[metabase.metabot.tools.shared :as shared][metabase.util.i18n :refer [tru]][metabase.util.log :as log]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [potemkin :as p]))(set! *warn-on-reflection* true)+;; Declared in `tools.error` so that the declarations this namespace loads can budget against the same limit+(p/import-vars+ [tools.error+ max-output-length+ output-budget])+(def runtime-codes"Every error code the runtime itself produces, with the class each one takes.@@ -175,19 +182,6 @@". Tool errors may only carry authored text; log the exception instead."){:where where})))))-(def max-output-length- "Maximum number of characters in a rendered failure, including recovery steps.-- Leave room for recovery steps when including excerpts."- 4000)--(defn output-budget- "Return the character budget for an excerpt after `prefix` and every recovery step."- [prefix recovery]- (let [tool-names (into #{} (mapcat :uses) recovery)]- (- max-output-length- (count (tools.error/recoverable-text {:message prefix :recovery recovery} tool-names)))))-(defn render"The failure [[::outcome]] for a ToolError, given the tool names available this turn.diff --git a/src/metabase/metabot/tools/shared.clj b/src/metabase/metabot/tools/shared.cljindex 78694e86c68..0d036cc3c72 100644--- a/src/metabase/metabot/tools/shared.clj+++ b/src/metabase/metabot/tools/shared.clj@@ -49,9 +49,12 @@(:request-canceled-chan (current-memory)))(defn current-queries-state- "Returns the current queries state map from agent memory."- []- (get-in (current-memory) [:state :queries] {}))+ "Returns the current queries state map from agent memory.+ Reads the memory in `memory-atom`, which a tool gets in its ctx, or else the bound [[*memory-atom*]]."+ ([]+ (current-queries-state *memory-atom*))+ ([memory-atom]+ (get-in (some-> memory-atom deref) [:state :queries] {})))(defn current-charts-state"Returns the current charts state map from agent memory."diff --git a/src/metabase/metabot/tools/skills.clj b/src/metabase/metabot/tools/skills.cljindex ea342404787..cf2216eefe1 100644--- a/src/metabase/metabot/tools/skills.clj+++ b/src/metabase/metabot/tools/skills.clj@@ -7,34 +7,86 @@(:require[clojure.string :as str][metabase.metabot.skills :as skills]- [metabase.util.log :as log]- [metabase.util.malli :as mu]))+ [metabase.metabot.tools.core :as tools]+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]+ [metabase.util.log :as log]))(set! *warn-on-reflection* true)-(defn- load-one- "Resolve and format a single skill id, or an inline message when it is unknown- or not available to the current user."+(def ^:private max-echoed-id-chars+ "The failure repeats at most this many characters of the requested ID, which the model supplies."+ 100)++(defn- loadable-skill-ids+ "The IDs of the skills the current request may load, sorted."+ []+ (->> (skills/all-skills)+ (filter skills/skill-loadable?)+ (map (comp name :id))+ sort+ vec))++(defn- not-loadable-text+ "The sentence saying that the skill `id` can't be loaded here, without the skills that can."[id]- (let [skill (skills/get-skill-by-id-string id)]- (cond- (nil? skill)- (do (log/info "Unknown skill requested by load_skill")- (format "Unknown skill: \"%s\". Load only ids listed under \"Available skills\"." id))-- (not (skills/skill-loadable? skill))- (do (log/info "Skill not available to current request" {:id id})- (format "Skill \"%s\" is not available to you." id))-- :else- (format "<skill id=\"%s\">\n%s\n</skill>" (name (:id skill)) (:body skill)))))--(mu/defn ^{:tool-name "load_skill"}- load-skill-tool- "Load the full instructions for one or more skills listed under \"Available- skills\". Call this immediately before doing the related work; you may load- several skills at once. Load only what is relevant to the current task."- [{:keys [ids]} :- [:map {:closed true}- [:ids [:sequential- [:string {:description "A skill id, e.g. construct-notebook-query-core"}]]]]]- {:output (str/join "\n\n" (map load-one ids))})+ (str "No skill with ID " (pr-str (llm-shape/truncate id max-echoed-id-chars)) " can be loaded here."))++(tools/defrecoverable unknown-skill!+ "No skill has the requested ID, or the current request may not load it."+ {:payload [:map {:closed true}+ [:id :string]+ [:loadable [:sequential :string]]]}+ [{:keys [id loadable]}]+ ;; A missing skill and a hidden one get the same sentence, so the reply reveals nothing about hidden skills+ {:message (str (not-loadable-text id) " "+ (if (seq loadable)+ (str "Skill IDs you can load: " (str/join ", " loadable) ".")+ "No skills can be loaded here."))+ :recovery []})++(def ^:private skill-id-schema+ [:string {:description "A skill id, e.g. construct-notebook-query-core"}])++(defrecord LoadSkillTool []+ tools/Tool+ (declaration [_]+ {:name "load_skill"+ :description (str "Load the full instructions for a skill listed under \"Available skills\". Call this "+ "immediately before doing the related work. Load only what is relevant to the current task.")+ :args [:map {:closed true} [:id skill-id-schema]]})++ (handle [_ {:keys [id]} _ctx]+ (let [skill (skills/get-skill-by-id-string id)]+ (when-not (some-> skill skills/skill-loadable?)+ (if skill+ (log/info "Skill not available to current request" {:id id})+ (log/info "Unknown skill requested by load_skill"))+ (unknown-skill! {:id id :loadable (loadable-skill-ids)}))+ {:output (format "<skill id=\"%s\">\n%s\n</skill>" (name (:id skill)) (:body skill))}))++ tools/BatchedTool+ (batched-declaration [_ declared]+ (assoc declared+ :description (str "Load the full instructions for one or more skills listed under \"Available "+ "skills\". Call this immediately before doing the related work; you may load several "+ "skills at once. Load only what is relevant to the current task.")+ ;; The dialect preload's synthetic call sends `ids` too (`skills/dialect-preload-parts`)+ :args [:map {:closed true}+ [:ids [:sequential {:min 1 :error/message "must be an array of at least one skill id"}+ skill-id-schema]]]))++ (batched-args [_ {:keys [ids]}]+ (mapv (fn [id] {:id id}) ids))++ (compose [_ entries _ctx]+ ;; Only the first failure lists the loadable skills, so they appear once and survive the runtime's length limit+ (let [first-failed (first (keep-indexed (fn [i entry] (when (:failed? entry) i)) entries))]+ {:output (str/join "\n\n" (map-indexed (fn [i {:keys [failed? output error]}]+ (if (and failed? (not= i first-failed))+ (not-loadable-text (get-in error [:data :id]))+ output))+ entries))})))++(def load-skill-tool+ "`load_skill`: one skill through [[tools/handle]], several through [[tools/run-batched]]."+ (->LoadSkillTool))diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.cljindex bb24ad3fb93..03a0985f7b6 100644--- a/src/metabase/metabot/tools/sql.clj+++ b/src/metabase/metabot/tools/sql.clj@@ -7,6 +7,7 @@[metabase.metabot.tmpl :as te][metabase.metabot.tools.core :as tools][metabase.metabot.tools.run-query :as tools.run-query]+ [metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape][metabase.metabot.tools.sql.common :as sql.common]@@ -159,7 +160,7 @@{:query-id query-id:edits edits:checklist checklist- :queries-state (get-in (some-> (:memory-atom ctx) deref) [:state :queries] {})})+ :queries-state (shared/current-queries-state (:memory-atom ctx))})title instructions/edit-sql-query-instructions-for ctx)))(def edit-sql-query-tool@@ -191,7 +192,7 @@{:query-id query-id:sql new-query:checklist checklist- :queries-state (get-in (some-> (:memory-atom ctx) deref) [:state :queries] {})})+ :queries-state (shared/current-queries-state (:memory-atom ctx))})title instructions/replace-sql-query-instructions-for ctx)))(def replace-sql-query-tooldiff --git a/src/metabase/metabot/tools/timelines.clj b/src/metabase/metabot/tools/timelines.cljindex 0913ff1b845..61d5da0a9a1 100644--- a/src/metabase/metabot/tools/timelines.clj+++ b/src/metabase/metabot/tools/timelines.clj@@ -1,31 +1,15 @@(ns metabase.metabot.tools.timelines- "`list_timelines` and `get_timeline_details`.-- `get_timeline_details` is converted: it is a record implementing- `metabase.metabot.tools.core/Tool`. `list_timelines` is not yet, so it is still an `mu/defn` var.- Both appear in the same profile; see `metabase.metabot.tools/->entries`."+ "`list_timelines` and `get_timeline_details`."(:require[clojure.string :as str][metabase.metabot.scope :as scope][metabase.metabot.tools.core :as tools][metabase.metabot.tools.recoverable.common :as recoverable.common][metabase.metabot.tools.shared.llm-shape :as llm-shape]- [metabase.metabot.tools.util :as metabot.tools.u]- [metabase.timeline.core :as timeline]- [metabase.util.malli :as mu]))+ [metabase.timeline.core :as timeline]))(set! *warn-on-reflection* true)-(defn get-timelines- "Lists timelines visible to the current user."- [_args]- (try- {:structured_output- (->> (timeline/list-timelines)- (map #(select-keys % [:id :name :description])))}- (catch Exception e- (metabot.tools.u/handle-agent-error e))))-(defn get-timeline-details"A timeline by id, including its events, or nil when there is no such timeline.@@ -67,22 +51,21 @@"<events />\n")"</timeline>"))-(defn- add-output- "Add :output to a tool result."- [result format-fn]- (if-let [structured (or (:structured_output result) (:structured-output result))]- (assoc result :output (format-fn structured))- result))--(mu/defn ^{:tool-name "list_timelines"- :scope scope/agent-timelines-read}- list-timelines-tool- "List all timelines available in the Metabase instance.+(defrecord ListTimelinesTool []+ tools/Tool+ (declaration [_]+ {:name "list_timelines"+ :description (str "List all timelines available in the Metabase instance.\n\n"+ "If a timeline looks relevant to the user's request, fetch its events using the "+ "get_timeline_details tool.")+ :scope scope/agent-timelines-read+ :args [:map {:closed true}]})+ (handle [_ _args _ctx]+ {:output (format-timeline-list-output (timeline/list-timelines))}))- If a timeline looks relevant to the user's request, fetch its events using the- get_timeline_details tool."- [_args :- [:map {:closed true}]]- (add-output (get-timelines {}) format-timeline-list-output))+(def list-timelines-tool+ "The `list_timelines` tool."+ (->ListTimelinesTool))(defrecord GetTimelineDetailsTool []tools/Tooldiff --git a/test/metabase/metabot/tools/autogen_dashboard_test.clj b/test/metabase/metabot/tools/autogen_dashboard_test.cljindex cf97e6b1f44..85c6bd98290 100644--- a/test/metabase/metabot/tools/autogen_dashboard_test.clj+++ b/test/metabase/metabot/tools/autogen_dashboard_test.clj@@ -2,108 +2,48 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase.api-scope.core :as api-scope][metabase.api.macros :as api.macros][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.metabot.agent.core :as agent][metabase.metabot.context :as metabot.context]- [metabase.metabot.test-util :as test-util]+ [metabase.metabot.db :as metabot.db]+ [metabase.metabot.scope :as scope]+ [metabase.metabot.tools :as metabot.tools][metabase.metabot.tools.autogen-dashboard :as autogen-dashboard]- [metabase.metabot.tools.legacy :as tools.legacy]- [metabase.metabot.tools.shared :as shared]+ [metabase.metabot.tools.recoverable.common :as recoverable.common][metabase.test :as mt][metabase.util.log.capture :as log.capture]))-;;; Source Resolution Tests (without database)+(defn- invoke+ "Call `create_autogenerated_dashboard` with `source` the way the agent loop does, in a conversation that holds+ `queries`, in a profile that also has `other-tools` (`search` by default). Return the result, or `{:output <the+ text the model reads> :error {:class :code}}` when the call fails."+ ([source] (invoke source {}))+ ([source queries] (invoke source queries [#'metabot.tools/search-tool]))+ ([source queries other-tools]+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]+ (metabot.tools/->entries (cons #'metabot.tools/create-autogenerated-dashboard-tool other-tools)+ (atom {:state {:queries queries}})+ nil+ :internal))]+ ((:fn (get entries "create_autogenerated_dashboard")) {:source source}))))-(deftest resolve-source-validation-test- (testing "fails when no source specified"- (let [memory-atom (atom {:state {:queries {} :charts {}}})]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Set exactly one of"- (autogen-dashboard/create-autogenerated-dashboard- {:source {}- :memory-atom memory-atom})))))- (testing "fails when query_id not found in memory"- (let [memory-atom (atom {:state {:queries {}}})]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Query not found in memory"- (autogen-dashboard/create-autogenerated-dashboard- {:source {:query_id "nonexistent"}- :memory-atom memory-atom}))))))+(defn- dashboard+ "The result of a call that created the dashboard titled `title` at `url`."+ [title url]+ {:output (str "The dashboard has been created and the user has been navigated to view it. Briefly "+ "acknowledge this (1 sentence) and stop. Do not describe the dashboard contents.")+ :data-parts [{:type :data+ :data-type "generated_entity"+ :data {:type "dashboard" :url url :title title}}]})-;;; Response Structure Tests--(deftest response-structure-test- (mt/with-current-user (mt/user->id :rasta)- (testing "returns instructions for LLM"- ;; We need to test the structure without actually hitting the database- ;; Using mt/with-temp for table creation- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id (:id table)}- :memory-atom (atom {:state {}})})]- (is (string? (:instructions result))))))- (testing "returns a dashboard generated_entity data part"- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id (:id table)}- :memory-atom (atom {:state {}})})- part (first (:data-parts result))- entity (:data part)]- (is (contains? result :data-parts))- (is (= "generated_entity" (:data-type part)))- (is (= "dashboard" (:type entity)))- (is (str/starts-with? (:url entity) "/auto/dashboard/"))- (is (= (or (:display_name table) (:name table)) (:title entity))))))- (testing "returns user-friendly message"- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id (:id table)}- :memory-atom (atom {:state {}})})]- (is (string? (get-in result [:structured-output :message])))- (is (str/includes? (get-in result [:structured-output :message]) "dashboard")))))- (testing "returns path in structured output"- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id (:id table)}- :memory-atom (atom {:state {}})})]- (is (string? (get-in result [:structured-output :path]))))))))--;;; Path Generation Tests--(deftest path-generation-test- (mt/with-current-user (mt/user->id :rasta)- (testing "table source generates correct path"- (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id (:id table)}- :memory-atom (atom {:state {}})})]- (is (= (str "/auto/dashboard/table/" (:id table))- (get-in result [:structured-output :path]))))))- (testing "model source generates correct path"- (mt/with-temp [:model/Card model {:type :model :name "test_model"}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:model_id (:id model)}- :memory-atom (atom {:state {}})})]- (is (= (str "/auto/dashboard/model/" (:id model))- (get-in result [:structured-output :path]))))))- (testing "metric source generates correct path"- (mt/with-temp [:model/Card metric {:type :metric :name "test_metric"}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:metric_id (:id metric)}- :memory-atom (atom {:state {}})})]- (is (= (str "/auto/dashboard/question/" (:id metric))- (get-in result [:structured-output :path]))))))- (testing "report source generates correct path"- (mt/with-temp [:model/Card report {:type :question :name "test_report"}]- (let [result (autogen-dashboard/create-autogenerated-dashboard- {:source {:report_id (:id report)}- :memory-atom (atom {:state {}})})]- (is (= (str "/auto/dashboard/question/" (:id report))- (get-in result [:structured-output :path]))))))))+(defn- failure+ "A call that failed with the declared error `code`, and the `lines` that the model reads."+ [code & lines]+ {:output (str/join "\n" lines)+ :error {:class :recoverable :code code}})(defn- xray-page"Fetch the dashboard for `path` from the X-ray API as `:rasta`."@@ -115,17 +55,37 @@(let [mp (mt/metadata-provider)](lib/query mp (lib.metadata/table mp (mt/id :venues)))))+(deftest dashboard-for-each-source-test+ (mt/with-temp [:model/Table {table-id :id :as table} {:name (mt/random-name) :db_id (mt/id)}+ :model/Card {model-id :id} {:type :model :name "Test model"}+ :model/Card {metric-id :id} {:type :metric :name "Test metric"}+ :model/Card {report-id :id} {:type :question :name "Test report"}]+ (mt/with-current-user (mt/user->id :rasta)+ (testing "the model is told to acknowledge the dashboard in a sentence, and the frontend gets a link to it"+ (is (= (dashboard (or (:display_name table) (:name table)) (str "/auto/dashboard/table/" table-id))+ (invoke {:table_id table-id}))))+ (testing "each kind of source has its own X-ray path"+ (is (= (dashboard "Test model" (str "/auto/dashboard/model/" model-id))+ (invoke {:model_id model-id})))+ (is (= (dashboard "Test metric" (str "/auto/dashboard/question/" metric-id))+ (invoke {:metric_id metric-id})))+ (is (= (dashboard "Test report" (str "/auto/dashboard/question/" report-id))+ (invoke {:report_id report-id}))))+ (testing "a query from the conversation is analyzed as it is"+ (is (=? (dashboard "Dashboard" #"^/auto/dashboard/adhoc/.+")+ (invoke {:query_id "q123"} {"q123" (venues-query)})))))))+(deftest xray-page-test(let [venues (venues-query)](mt/with-temp [:model/Card {model-id :id} {:type :model :dataset_query venues}:model/Card {metric-id :id} {:type :metric :dataset_query (lib/aggregate venues (lib/count))}:model/Card {question-id :id} {:type :question :dataset_query venues}]- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" venues- "q-legacy" (lib/->legacy-MBQL venues)- "q-model" (lib/query (mt/metadata-provider)- (lib.metadata/card- (mt/metadata-provider) model-id))- "q-staged" (lib/append-stage venues)}}})]+ (let [queries {"q-1" venues+ "q-legacy" (lib/->legacy-MBQL venues)+ "q-model" (lib/query (mt/metadata-provider)+ (lib.metadata/card+ (mt/metadata-provider) model-id))+ "q-staged" (lib/append-stage venues)}](mt/with-current-user (mt/user->id :rasta)(doseq [source [{:table_id (mt/id :venues)}{:model_id model-id}@@ -137,69 +97,78 @@{:query_id "q-staged"}]](testing (str "the X-ray link for " (pr-str source) " returns a dashboard from the API")(is (=? {:dashcards seq}- (->> (autogen-dashboard/create-autogenerated-dashboard-tool {:source source})- :structured-output- :path+ (->> (invoke source queries)+ :data-parts first :data :url(xray-page 200)))))))))))(deftest log-test(testing "the log records the source and leaves out the query text"- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" (venues-query)}}})]+ (let [queries {"q-1" (venues-query)}](log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.autogen-dashboard :info]](mt/with-current-user (mt/user->id :rasta)- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-1"}}))+ (invoke {:query_id "q-1"} queries))(is (= ["Creating autogenerated dashboard {:source {:query_id q-1}}""Autogenerated dashboard created {:source {:query_id q-1}}"](map :message (logs))))))))-;;; Entity Not Found Tests+(deftest unknown-query-id-test+ (testing "a query ID that the conversation doesn't hold fails as it does for the other tools that take one"+ (is (= (failure ::recoverable.common/unknown-query-id+ "Query \"q-9\" was not found. Available query IDs: \"q-1\"."+ "Use an available query ID from this conversation.")+ (invoke {:query_id "q-9"} {"q-1" (venues-query)}))))+ (testing "with no query to use, the model is told to build a notebook query, since X-ray refuses a SQL one"+ (is (= (failure ::recoverable.common/unknown-query-id+ "Query \"q-9\" was not found. Available query IDs: none."+ "Use `construct_notebook_query` to build a query, then use the returned query ID.")+ (invoke {:query_id "q-9"} {} [#'metabot.tools/construct-notebook-query-tool+ #'metabot.tools/create-sql-query-tool]))))+ (testing "only the queries that X-ray can analyze are listed, not the SQL or incomplete ones that it refuses"+ (let [sql (lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES")]+ (is (= (failure ::recoverable.common/unknown-query-id+ "Query \"q-9\" was not found. Available query IDs: \"q-1\", \"q-legacy\"."+ "Use an available query ID from this conversation.")+ (invoke {:query_id "q-9"} {"q-1" (venues-query)+ "q-legacy" (lib/->legacy-MBQL (venues-query))+ "q-sql" sql+ "q-sql-legacy" (lib/->legacy-MBQL sql)+ "q-sql-staged" (lib/append-stage sql)+ "q-draft" {:database (mt/id)}})))))+ (testing "a long list of IDs is cut to complete IDs, so the recovery step still reaches the model"+ (let [{:keys [output error]} (invoke {:query_id "q-9"}+ (zipmap (repeatedly 100 #(str (random-uuid))) (repeat (venues-query))))]+ (is (= {:class :recoverable :code ::recoverable.common/unknown-query-id} error))+ (is (re-find #"\" and \d+ more\.\nUse an available query ID from this conversation\.$" output)))))-(deftest entity-not-found-test- (testing "fails for nonexistent table"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Table 999999 was not found"- (autogen-dashboard/create-autogenerated-dashboard- {:source {:table_id 999999}- :memory-atom (atom {:state {}})}))))- (testing "fails for nonexistent model"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Model 999999 was not found"- (autogen-dashboard/create-autogenerated-dashboard- {:source {:model_id 999999}- :memory-atom (atom {:state {}})}))))- (testing "fails for nonexistent metric"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Metric 999999 was not found"- (autogen-dashboard/create-autogenerated-dashboard- {:source {:metric_id 999999}- :memory-atom (atom {:state {}})}))))- (testing "fails for nonexistent report"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Question 999999 was not found"- (autogen-dashboard/create-autogenerated-dashboard- {:source {:report_id 999999}- :memory-atom (atom {:state {}})})))))--(defn- failure- "How a `create_autogenerated_dashboard` call with `source` fails, as the agent sees it."- [source]- (test-util/tool-failure (tools.legacy/adapt #'autogen-dashboard/create-autogenerated-dashboard-tool)- {:source source}))+(deftest one-source-test+ (testing "a call that sets no source, or two, is rejected before any source is read"+ (doseq [source [{} {:table_id 1 :model_id 2}]]+ (is (= (failure ::autogen-dashboard/invalid-source+ "Set exactly one of `table_id`, `model_id`, `metric_id`, `query_id` or `report_id` in `source`.")+ (invoke source)))))+ (testing "a placeholder for an unused source is not a second source"+ (mt/with-temp [:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id)}+ :model/Card {model-id :id} {:type :model}]+ (mt/with-current-user (mt/user->id :rasta)+ (doseq [[source path] [[{:table_id nil :model_id model-id} (str "/auto/dashboard/model/" model-id)]+ [{:table_id table-id :model_id 0} (str "/auto/dashboard/table/" table-id)]+ [{:table_id 0 :model_id model-id} (str "/auto/dashboard/model/" model-id)]+ [{:query_id "" :table_id table-id} (str "/auto/dashboard/table/" table-id)]]]+ (testing (pr-str source)+ (is (=? {:data-parts [{:data {:url path}}]}+ (invoke source)))))))))-(deftest create-autogenerated-dashboard-tool-errors-test- (testing "a query missing from memory goes back to the agent as a failed call"- (is (= {:class :recoverable- :code :metabase.metabot.tools.recoverable.legacy/agent-error- :text "Query not found in memory with ID: q-9. Available queries: []"}- (failure {:query_id "q-9"}))))- (testing "an unexpected error propagates to the agent loop"- (mt/with-dynamic-fn-redefs [autogen-dashboard/create-autogenerated-dashboard (fn [_] (throw (ex-info "boom" {})))]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:table_id 1}}))))))+(deftest missing-source-test+ (mt/with-current-user (mt/user->id :rasta)+ (doseq [[source-key noun] {:table_id "Table"+ :model_id "Model"+ :metric_id "Metric"+ :report_id "Question"}]+ (testing noun+ (is (= (failure ::recoverable.common/not-found+ (str noun " 999999 was not found. It may not exist, or you may not have access to it.")+ "Call `search` to find the entity you want and use an id from the results.")+ (invoke {source-key 999999})))))))(deftest unreadable-source-test(mt/with-non-admin-groups-no-root-collection-perms@@ -215,32 +184,30 @@[:table_id table-id "Table"]]](testing noun(testing "fails for a user who cannot read it, the same way a missing one does"- (is (=? {:class :recoverable- :code :metabase.metabot.tools.recoverable.common/not-found- :text (re-pattern (str "(?s)" noun " " id " was not found\\..*"))}- (mt/with-current-user (mt/user->id :rasta)- (failure {source-key id})))))+ (is (= (failure ::recoverable.common/not-found+ (str noun " " id " was not found. It may not exist, or you may not have access to it.")+ "Call `search` to find the entity you want and use an id from the results.")+ (mt/with-current-user (mt/user->id :rasta)+ (invoke {source-key id})))))(testing "still gets an admin a dashboard"(is (=? {:data-parts [{:data {:title #"Secret .*"}}]}(mt/with-current-user (mt/user->id :crowberto)- (autogen-dashboard/create-autogenerated-dashboard-tool {:source {source-key id}})))))))))))+ (invoke {source-key id})))))))))))(deftest unrunnable-query-test(testing "a user without query permissions gets no X-ray link"- (binding [shared/*memory-atom* (atom {:state {:queries {"q-1" (venues-query)}}})]- (let [path (mt/with-current-user (mt/user->id :crowberto)- (-> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-1"}})- :structured-output- :path))]- (mt/with-no-data-perms-for-all-users!- (is (= "You don't have permissions to do that."- (xray-page 403 path)))- (is (= {:class :recoverable- :code :metabase.metabot.tools.recoverable.common/not-found- :text (str "Query q-1 was not found. It may not exist, or you may not have access to it.\n"- "Call `search` to find the entity you want and use an id from the results.")}- (mt/with-current-user (mt/user->id :rasta)- (failure {:query_id "q-1"})))))))))+ (let [queries {"q-1" (venues-query)}+ path (mt/with-current-user (mt/user->id :crowberto)+ (->> (invoke {:query_id "q-1"} queries)+ :data-parts first :data :url))]+ (mt/with-no-data-perms-for-all-users!+ (is (= "You don't have permissions to do that."+ (xray-page 403 path)))+ (is (= (failure ::recoverable.common/not-found+ "Query q-1 was not found. It may not exist, or you may not have access to it."+ "Call `search` to find the entity you want and use an id from the results.")+ (mt/with-current-user (mt/user->id :rasta)+ (invoke {:query_id "q-1"} queries))))))))(deftest unreadable-joined-card-test(mt/with-non-admin-groups-no-root-collection-perms@@ -252,20 +219,19 @@query (-> (venues-query)(lib/join (lib.metadata/card mp card-id)))](doseq [query [query (lib/->legacy-MBQL query)]]- (binding [shared/*memory-atom* (atom {:state {:queries {"q-join" query}}})]- (let [path (mt/with-current-user (mt/user->id :crowberto)- (-> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-join"}})- :structured-output :path))]- (testing "the X-ray API denies a query joining a card the user cannot read"- (is (= "You don't have permissions to do that."- (xray-page 403 path))))- (testing "the tool refuses the query before returning a link"- (is (= {:class :recoverable- :code :metabase.metabot.tools.recoverable.common/not-found- :text (str "Query q-join was not found. It may not exist, or you may not have access to it.\n"- "Call `search` to find the entity you want and use an id from the results.")}- (mt/with-current-user (mt/user->id :rasta)- (failure {:query_id "q-join"}))))))))))))+ (let [queries {"q-join" query}+ path (mt/with-current-user (mt/user->id :crowberto)+ (->> (invoke {:query_id "q-join"} queries)+ :data-parts first :data :url))]+ (testing "the X-ray API denies a query joining a card the user cannot read"+ (is (= "You don't have permissions to do that."+ (xray-page 403 path))))+ (testing "the tool refuses the query before returning a link"+ (is (= (failure ::recoverable.common/not-found+ "Query q-join was not found. It may not exist, or you may not have access to it."+ "Call `search` to find the entity you want and use an id from the results.")+ (mt/with-current-user (mt/user->id :rasta)+ (invoke {:query_id "q-join"} queries)))))))))))(deftest draft-query-test(testing "a draft query from the viewing context needs a data source for X-ray"@@ -274,13 +240,11 @@(let [context (api.macros/decode-and-validate-params:body ::metabot.context/context{:user_is_viewing [{:id "q-draft" :type "adhoc" :query draft}]})]- (binding [shared/*memory-atom* (atom {:state (#'agent/seed-state {} context)})]- (is (= {:class :recoverable- :code ::autogen-dashboard/incomplete-query- :text (str "Query q-draft has no data source yet, so X-ray cannot analyze it.\n"- "Call `search` to find a table, model or metric to X-ray instead.")}- (mt/with-current-user (mt/user->id :crowberto)- (failure {:query_id "q-draft"}))))))))))+ (is (= (failure ::autogen-dashboard/incomplete-query+ "Query q-draft has no data source yet, so X-ray cannot analyze it."+ "Call `search` to find a table, model or metric to X-ray instead.")+ (mt/with-current-user (mt/user->id :crowberto)+ (invoke {:query_id "q-draft"} (:queries (#'agent/seed-state {} context)))))))))))(deftest sourceless-query-test(testing "selecting a database without a table or card is not enough for an X-ray"@@ -291,61 +255,37 @@{:database (mt/id) :lib/type :mbql/query:stages [{:lib/type :mbql.stage/mbql :source-table nil}]}]](testing (pr-str draft)- (binding [shared/*memory-atom* (atom {:state {:queries {"q-draft" draft}}})]- (is (= {:class :recoverable- :code ::autogen-dashboard/incomplete-query- :text (str "Query q-draft has no data source yet, so X-ray cannot analyze it.\n"- "Call `search` to find a table, model or metric to X-ray instead.")}- (mt/with-current-user (mt/user->id :crowberto)- (failure {:query_id "q-draft"})))))))))+ (is (= (failure ::autogen-dashboard/incomplete-query+ "Query q-draft has no data source yet, so X-ray cannot analyze it."+ "Call `search` to find a table, model or metric to X-ray instead.")+ (mt/with-current-user (mt/user->id :crowberto)+ (invoke {:query_id "q-draft"} {"q-draft" draft}))))))))(deftest unavailable-source-test(mt/with-temp [:model/Card {model-id :id} {:type :model :archived true}:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id) :active false}](mt/with-current-user (mt/user->id :rasta)(testing "an archived source the user can read is in the trash"- (is (= {:class :recoverable- :code ::autogen-dashboard/unavailable-source- :text (str "Model " model-id " is in the trash.\n"- "Call `search` to find another source and use an id from the results.")}- (failure {:model_id model-id}))))+ (is (= (failure ::autogen-dashboard/unavailable-source+ (str "Model " model-id " is in the trash.")+ "Call `search` to find another source and use an id from the results.")+ (invoke {:model_id model-id}))))(testing "an inactive table is gone from its database"- (is (= {:class :recoverable- :code ::autogen-dashboard/unavailable-source- :text (str "Table " table-id " is no longer in its database.\n"- "Call `search` to find another source and use an id from the results.")}- (failure {:table_id table-id})))))))--(deftest one-source-test- (testing "a call that sets two sources is rejected before either is read"- (is (= {:class :recoverable- :code ::autogen-dashboard/invalid-source- :text "Set exactly one of `table_id`, `model_id`, `metric_id`, `query_id` or `report_id` in `source`."}- (failure {:table_id 1 :model_id 2}))))- (testing "a placeholder for an unused source is not a second source"- (mt/with-temp [:model/Table {table-id :id} {:name (mt/random-name) :db_id (mt/id)}- :model/Card {model-id :id} {:type :model}]- (mt/with-current-user (mt/user->id :rasta)- (doseq [[source path] [[{:table_id nil :model_id model-id} (str "/auto/dashboard/model/" model-id)]- [{:table_id table-id :model_id 0} (str "/auto/dashboard/table/" table-id)]- [{:table_id 0 :model_id model-id} (str "/auto/dashboard/model/" model-id)]- [{:query_id "" :table_id table-id} (str "/auto/dashboard/table/" table-id)]]]- (testing (pr-str source)- (is (=? {:structured-output {:path path}}- (autogen-dashboard/create-autogenerated-dashboard-tool {:source source})))))))))+ (is (= (failure ::autogen-dashboard/unavailable-source+ (str "Table " table-id " is no longer in its database.")+ "Call `search` to find another source and use an id from the results.")+ (invoke {:table_id table-id})))))))(deftest native-query-test(testing "an unsaved SQL query gets a recoverable refusal instead of a broken X-ray link"(doseq [query ((juxt identity lib/->legacy-MBQL)(lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES"))](testing (pr-str (dissoc query :lib/metadata))- (binding [shared/*memory-atom* (atom {:state {:queries {"q-sql" query}}})]- (is (= {:class :recoverable- :code ::autogen-dashboard/unsupported-native-query- :text (str "Query q-sql is an unsaved SQL query, which X-ray does not support.\n"- "Call `search` to find a table, model, metric or saved question to X-ray instead.")}- (mt/with-current-user (mt/user->id :rasta)- (failure {:query_id "q-sql"})))))))))+ (is (= (failure ::autogen-dashboard/unsupported-native-query+ "Query q-sql is an unsaved SQL query, which X-ray does not support."+ "Call `search` to find a table, model, metric or saved question to X-ray instead.")+ (mt/with-current-user (mt/user->id :rasta)+ (invoke {:query_id "q-sql"} {"q-sql" query}))))))))(deftest saved-native-query-test(testing "a saved SQL question has a working X-ray link"@@ -354,18 +294,22 @@"SELECT * FROM VENUES")})](mt/with-current-user (mt/user->id :rasta)(is (=? {:dashcards seq}- (->> (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:report_id card-id}})- :structured-output :path+ (->> (invoke {:report_id card-id})+ :data-parts first :data :url(xray-page 200))))))))(deftest unreadable-native-query-test(testing "a SQL query that the user cannot run returns a not-found error"- (binding [shared/*memory-atom* (atom {:state {:queries {"q-sql" (lib/native-query (mt/metadata-provider)- "SELECT * FROM VENUES")}}})]+ (let [queries {"q-sql" (lib/native-query (mt/metadata-provider) "SELECT * FROM VENUES")}](mt/with-no-data-perms-for-all-users!- (is (= {:class :recoverable- :code :metabase.metabot.tools.recoverable.common/not-found- :text (str "Query q-sql was not found. It may not exist, or you may not have access to it.\n"- "Call `search` to find the entity you want and use an id from the results.")}+ (is (= (failure ::recoverable.common/not-found+ "Query q-sql was not found. It may not exist, or you may not have access to it."+ "Call `search` to find the entity you want and use an id from the results.")(mt/with-current-user (mt/user->id :rasta)- (failure {:query_id "q-sql"}))))))))+ (invoke {:query_id "q-sql"} queries))))))))++(deftest unexpected-error-test+ (testing "an error that no one declared ends the turn"+ (mt/with-dynamic-fn-redefs [metabot.db/table (fn [_] (throw (ex-info "boom" {})))]+ (is (= {:class :unrecoverable :code :internal}+ (:error (invoke {:table_id 1})))))))diff --git a/test/metabase/metabot/tools/entity_retrieval_test.clj b/test/metabase/metabot/tools/entity_retrieval_test.cljindex 2ad6fc91d9f..921e02f7e2e 100644--- a/test/metabase/metabot/tools/entity_retrieval_test.clj+++ b/test/metabase/metabot/tools/entity_retrieval_test.clj@@ -2,6 +2,10 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase.api-scope.core :as api-scope]+ [metabase.metabot.scope :as scope]+ [metabase.metabot.tools :as metabot.tools]+ [metabase.metabot.tools.core :as tools.core][metabase.metabot.tools.entity-retrieval :as entity-retrieval][metabase.metabot.tools.search :as tools.search][metabase.search.entity-retrieval.core :as cs.core]@@ -18,6 +22,19 @@`(mt/with-dynamic-fn-redefs [cs.core/search-unfiltered ~search-fn]~@body))+(defn- retrieve+ "Call `retrieve_library_entities` with `args`."+ [args]+ (tools.core/handle entity-retrieval/retrieve-library-entities-tool args {}))++(defn- invoke+ "Call `retrieve_library_entities` with `args` the way the agent loop does. Return the result, or+ `{:output <the text the model reads> :error {:class :code}}` when the call fails."+ [args]+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]+ (metabot.tools/->entries [#'metabot.tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]+ ((:fn (get entries "retrieve_library_entities")) args)))+(deftest tool-shape-oss-fallback-test(testing "without the library-retrieval feature the tool returns the standard empty result shape";; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient@@ -28,8 +45,7 @@:data []:total_count 0:weak_match false}}- (entity-retrieval/retrieve-library-entities-tool- {:user_search_prompt "revenue per region"}))))))+ (retrieve {:user_search_prompt "revenue per region"}))))))(deftest similarity-helper-test(let [similarity (var-get #'entity-retrieval/similarity)]@@ -75,8 +91,7 @@:doc_text sentinel:score {:total_score 0.9:scores [{:name :similarity :score 0.9}]}}]- search! #(entity-retrieval/retrieve-library-entities-tool- {:user_search_prompt "secret"})]+ search! #(retrieve {:user_search_prompt "secret"})](with-unfiltered-search (fn [_ _] raw)(mt/with-dynamic-fn-redefs[cs.core/library-entity-keys (constantly #{["model" secret-id]})]@@ -199,5 +214,13 @@(testing "a failing library search propagates to the agent loop, which reports the call as failed"(with-unfiltered-search (fn [_ _] (throw (ex-info "Embedding service unavailable" {})))(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Embedding service unavailable"- (entity-retrieval/retrieve-library-entities-tool- {:user_search_prompt "revenue per region"}))))))+ (retrieve {:user_search_prompt "revenue per region"}))))))++(deftest ^:parallel blank-search-prompt-test+ (testing "a blank prompt goes back to the model before anything is searched"+ (with-unfiltered-search (fn [_ _] (throw (ex-info "searched" {})))+ (doseq [prompt ["" " "]]+ (is (= {:output (str "`user_search_prompt` is blank. Set it to a full sentence that describes the data the "+ "user wants.")+ :error {:class :recoverable :code ::entity-retrieval/blank-search-prompt}}+ (invoke {:user_search_prompt prompt})))))))diff --git a/test/metabase/metabot/tools/run_query_test.clj b/test/metabase/metabot/tools/run_query_test.cljindex 334d0de648d..ed704d8ccc7 100644--- a/test/metabase/metabot/tools/run_query_test.clj+++ b/test/metabase/metabot/tools/run_query_test.clj@@ -11,6 +11,7 @@[metabase.metabot.scope :as scope][metabase.metabot.tools :as tools][metabase.metabot.tools.core :as tools.core]+ [metabase.metabot.tools.recoverable.common :as recoverable.common][metabase.metabot.tools.run-query :as run-query][metabase.metabot.tools.sql.common :as sql.common][metabase.permissions.core :as perms]@@ -170,9 +171,9 @@(is (= (refusal ::run-query/query-execution-off "Query execution is turned off for Metabot.")(call-tool! {:state {:queries {"q1" (venues-by-id)}}} {:query_id "q1"})))))(testing "an unknown id lists the ids the model can use"- (is (= (refusal ::run-query/unknown-query-id- (str "No query with ID nope. Known query IDs: [q1]. Use the ID of a query that you built, or a "- "saved question or model that the user can read."))+ (is (= (refusal ::recoverable.common/unknown-query-id+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". Use the ID of a query that you "+ "built, or a saved question or model that the user can read."))(run-tool! {"q1" (venues-by-id)} {:query_id "nope"}))))(testing "a SQL query is refused with a pointer to construct_notebook_query while SQL execution is off"(mt/with-temporary-setting-values [metabot-sql-execution-enabled? false]@@ -522,8 +523,8 @@(data-lines (:output (run-viewing! [] (str notebook-card)))))))(testing "an id with anything but digits names no saved question"(doseq [id [(str "+" notebook-card) (str notebook-card " ") (str "0" notebook-card)]]- (is (=? {:output #"No query with ID .*\. Known query IDs: \[\]\. .*"- :error {:code ::run-query/unknown-query-id}}+ (is (=? {:output #"Query \".*\" was not found\. Available query IDs: none\. .*"+ :error {:code ::recoverable.common/unknown-query-id}}(run-viewing! [] id)))))(testing "a viewed question the user can't read is refused like an id that names nothing"(mt/with-non-admin-groups-no-root-collection-perms@@ -865,8 +866,8 @@(call-tool-in-conversation! {:state {:queries {}}:context {:user_is_viewing viewing}}{:query_id query-id})))- unknown-id {:output #"No query with ID -?\d+\. Known query IDs: \[\]\. .*"- :error {:code ::run-query/unknown-query-id}}+ unknown-id {:output #"Query \"-?\d+\" was not found\. Available query IDs: none\. .*"+ :error {:code ::recoverable.common/unknown-query-id}}metric-refusal (fn [card-id step](refusal ::run-query/metric-not-runnable(str "Query ID " card-id " is a metric, and a metric does not run by its ID.")@@ -969,8 +970,9 @@(deftest run-sql-query-tool-refusals-test(testing "the SQL-only tool's refusals point at SQL or at telling the user, never at the notebook builder"(testing "an unknown id"- (is (= (refusal ::run-query/unknown-query-id- "No query with ID nope. Known query IDs: [q1]. A saved question or model has no query ID."+ (is (= (refusal ::recoverable.common/unknown-query-id+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". A saved question or model has "+ "no query ID.")(str "To run one, write SQL that reads it with `create_sql_query`, then run that query with ""`run_query`."))(run-sql-only-tool! {"q1" (venues-by-id)} {:query_id "nope"}))))@@ -999,9 +1001,9 @@(is (= (refusal ::run-query/sql-not-allowed "This query is SQL, and SQL can't run here." no-values-step)(run-sql-only-tool! false {"q1" (mt/native-query {:query "SELECT 1"})} {:query_id "q1"}))))(testing "an unknown id"- (is (= (refusal ::run-query/unknown-query-id- (str "No query with ID nope. Known query IDs: [q1]. A saved question or model has no query ID, "- "and no SQL can run here.")+ (is (= (refusal ::recoverable.common/unknown-query-id+ (str "Query \"nope\" was not found. Available query IDs: \"q1\". A saved question or model has "+ "no query ID, and no SQL can run here.")no-values-step)(run-sql-only-tool! false {"q1" (venues-by-id)} {:query_id "nope"}))))(testing "a query that is not SQL"@@ -1014,26 +1016,34 @@(is (=? {:structured-output {:query-id "q1" :returned 1}}(run-sql-only-tool! {"q1" (mt/native-query {:query "SELECT COUNT(*) FROM VENUES"})} {:query_id "q1"})))))-(deftest run-sql-query-tool-many-query-ids-test- (testing "a long list of known IDs is cut to leave room for the recovery step"- (let [ids (repeatedly 200 u/generate-nano-id)- {:keys [output error]} (run-sql-only-tool! (zipmap ids (repeat (venues-by-id))) {:query_id "nope"})- [message step] (str/split-lines output)- listed (some-> (re-find #"Known query IDs: \[([^\]]*)\] and 100 more\." message)- second- (str/split #", "))]- (is (= ::run-query/unknown-query-id (:code error)))- (is (= 100 (count listed)))- (is (every? (set ids) listed))- (is (= "To run one, write SQL that reads it with `create_sql_query`, then run that query with `run_query`."- step)))))+(deftest run-query-many-query-ids-test+ (let [built-or-saved (str "Use the ID of a query that you built, or a saved question or model that the user can "+ "read.")+ write-sql (str "To run one, write SQL that reads it with `create_sql_query`, then run that query with "+ "`run_query`.")]+ (testing "a long list of known IDs is cut to complete IDs, so the advice after it survives the runtime's cut"+ (doseq [[id-kind ids] {"NanoIDs" (repeatedly 200 u/generate-nano-id)+ "UUIDs" (repeatedly 100 #(str (random-uuid)))}+ [profile run advice] [["ordinary" run-tool! built-or-saved]+ ["SQL-only" run-sql-only-tool! write-sql]]]+ (testing (str profile " run_query, " id-kind)+ (let [{:keys [output error]} (run (zipmap ids (repeat (venues-by-id))) {:query_id "nope"})+ [_ shown more] (re-find #"Available query IDs: ([^.]*?)(?: and (\d+) more)?\." output)+ listed (some->> shown (re-seq #"\"([^\"]*)\"") (map second))]+ (is (= ::recoverable.common/unknown-query-id (:code error)))+ (is (str/includes? output advice))+ (is (every? (set ids) listed))+ (is (= (- (count ids) (count listed)) (some-> more parse-long)))))))+ (testing "an ID too long to fit is counted rather than cut"+ (is (str/includes? (:output (run-tool! {(apply str (repeat 5000 "y")) (venues-by-id)} {:query_id "nope"}))+ (str "Available query IDs: 1, too long to list. " built-or-saved))))))(deftest run-sql-query-tool-huge-query-id-test(testing "an invented ID is cut to leave room for the recovery step"(let [query-id (apply str (repeat 5000 "x"))]- (is (= (refusal ::run-query/unknown-query-id- (str "No query with ID " (subs query-id 0 100) ".... Known query IDs: [q1]. A saved question or "- "model has no query ID.")+ (is (= (refusal ::recoverable.common/unknown-query-id+ (str "Query \"" (subs query-id 0 100) "\" was not found. Available query IDs: \"q1\". A saved "+ "question or model has no query ID.")(str "To run one, write SQL that reads it with `create_sql_query`, then run that query with ""`run_query`."))(run-sql-only-tool! {"q1" (venues-by-id)} {:query_id query-id}))))))@@ -1078,8 +1088,8 @@metabot-sql-execution-enabled? true](binding [scope/*current-user-metabot-permissions* permsscope/*current-user-scope* (scope/user-metabot-perms->scopes perms)]- (is (=? {:output #"(?s)No query with ID \d+\. .*write SQL that reads it with `create_sql_query`.*"- :error {:code ::run-query/unknown-query-id}}+ (is (=? {:output #"(?s)Query \"\d+\" was not found\. .*write SQL that reads it with `create_sql_query`.*"+ :error {:code ::recoverable.common/unknown-query-id}}(call-sql-tool-in-conversation! {:state {:queries {}}:context {:user_is_viewing [{:type "question", :id card-id}]}}{:query_id (str card-id)})))))))))diff --git a/test/metabase/metabot/tools/skills_test.clj b/test/metabase/metabot/tools/skills_test.cljindex daf52961ac2..ed772a20389 100644--- a/test/metabase/metabot/tools/skills_test.clj+++ b/test/metabase/metabot/tools/skills_test.clj@@ -2,40 +2,106 @@(:require[clojure.string :as str][clojure.test :refer [deftest is testing]]+ [metabase.metabot.scope :as scope][metabase.metabot.skills :as skills]+ [metabase.metabot.tools :as metabot.tools]+ [metabase.metabot.tools.core :as tools.core]+ [metabase.metabot.tools.error :as tools.error]+ [metabase.metabot.tools.runtime :as tools.runtime][metabase.metabot.tools.skills :as tools.skills]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.log.capture :as log.capture]+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)-(deftest ^:parallel load-skill-tool-metadata-test- (testing "the tool var carries the metadata profiles require"- (let [m (meta #'tools.skills/load-skill-tool)]- (is (= "load_skill" (:tool-name m)))- (is (some? (:schema m))))))+(defn- load-skills+ "Call `load_skill` for `ids` the way the agent loop does, in a request whose skill manifest holds `loadable`.+ Return the result, or `{:output <the text the model reads> :error {:class :code}}` when the call fails."+ [loadable ids]+ (binding [scope/*current-loadable-skill-ids* (atom (set loadable))]+ (let [entries (metabot.tools/->entries [#'metabot.tools/load-skill-tool] (atom {}) nil :internal)]+ ((:fn (get entries "load_skill")) {:ids ids}))))++(deftest ^:parallel load-skill-takes-ids-test+ (testing "the model sends `ids`, and so does the synthetic call that preloads a SQL dialect"+ (let [{:keys [args]} (tools.core/validate-tool! tools.skills/load-skill-tool)+ [preload] (skills/dialect-preload-parts "postgresql")]+ (is (= "load_skill" (:function preload)))+ (is (mr/validate args (:arguments preload))))))(deftest ^:parallel load-skill-returns-body-test(testing "loading a known skill returns its body wrapped in a <skill> tag"- (let [out (:output (tools.skills/load-skill-tool {:ids ["construct-notebook-query-operators"]}))]- (is (str/includes? out "<skill id=\"construct-notebook-query-operators\">"))- (is (str/includes? out "</skill>"))- (is (str/includes? out "Aggregations")))))+ (let [{:keys [output error]} (load-skills [:construct-notebook-query-operators]+ ["construct-notebook-query-operators"])]+ (is (nil? error))+ (is (str/starts-with? output "<skill id=\"construct-notebook-query-operators\">\n"))+ (is (str/ends-with? output "\n</skill>"))+ (is (str/includes? output "Aggregations")))))(deftest ^:parallel load-skill-multiple-test- (testing "multiple ids are concatenated"- (let [out (:output (tools.skills/load-skill-tool- {:ids ["construct-notebook-query-core" "read-resource"]}))]- (is (str/includes? out "<skill id=\"construct-notebook-query-core\">"))- (is (str/includes? out "<skill id=\"read-resource\">")))))+ (testing "several skills come back in the order asked for, separated by a blank line"+ (let [{:keys [output]} (load-skills [:construct-notebook-query-core :read-resource]+ ["read-resource" "construct-notebook-query-core"])]+ (is (= ["read-resource" "construct-notebook-query-core"]+ (map second (re-seq #"<skill id=\"([^\"]+)\">" output))))+ (is (str/includes? output "</skill>\n\n<skill id=\"construct-notebook-query-core\">")))))(deftest ^:parallel load-skill-unknown-test- (testing "an unknown id yields an inline message, not an exception"- (let [out (:output (tools.skills/load-skill-tool {:ids ["does-not-exist"]}))]- (is (str/includes? out "Unknown skill"))- (is (str/includes? out "does-not-exist")))))+ (testing "a skill that doesn't exist and one that this request may not load fail the same way"+ (doseq [id ["does-not-exist" "edit-chart"]]+ (testing id+ (is (= {:output (str "No skill with ID \"" id "\" can be loaded here. Skill IDs you can load: "+ "construct-notebook-query-core, read-resource.")+ :error {:class :recoverable :code ::tools.core/all-items-failed}}+ (load-skills [:read-resource :construct-notebook-query-core] [id]))))))+ (testing "each skill that doesn't load is the declared error"+ (binding [scope/*current-loadable-skill-ids* (atom #{})]+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (tools.core/handle tools.skills/load-skill-tool {:id "read-resource"} {})))]+ (is (= ::tools.skills/unknown-skill (:code (tools.error/classify e)))))))+ (testing "when the request may load no skills, the model is told so"+ (is (= "No skill with ID \"read-resource\" can be loaded here. No skills can be loaded here."+ (:output (load-skills [] ["read-resource"])))))+ (testing "a long ID is cut short, so the list of skills stays within the failure text limit"+ (is (str/ends-with? (:output (load-skills [:read-resource] [(apply str (repeat 5000 "x"))]))+ "...\" can be loaded here. Skill IDs you can load: read-resource.")))+ (testing "the skills that load still reach the model, and the failure keeps its place"+ (let [{:keys [output error]} (load-skills [:read-resource] ["read-resource" "does-not-exist"])]+ (is (nil? error))+ (is (str/starts-with? output "<skill id=\"read-resource\">"))+ (is (str/ends-with? output (str "</skill>\n\nNo skill with ID \"does-not-exist\" can be loaded here. "+ "Skill IDs you can load: read-resource."))))))++(deftest ^:parallel load-skill-lists-loadable-skills-once-test+ (testing "several IDs that don't load list the skills that can be loaded once, after the first of them"+ (is (= {:output (str "No skill with ID \"nope-1\" can be loaded here. Skill IDs you can load: read-resource.\n\n"+ "No skill with ID \"nope-2\" can be loaded here.")+ :error {:class :recoverable :code ::tools.core/all-items-failed}}+ (load-skills [:read-resource] ["nope-1" "nope-2"]))))+ (testing "a call whose failures run past the failure text limit is cut short, and still lists the skills"+ (let [ids (for [i (range 40)] (str i (apply str (repeat 200 "x"))))+ {:keys [output error]} (load-skills [:read-resource] ids)]+ (is (= {:class :recoverable :code ::tools.core/all-items-failed} error))+ (is (= tools.runtime/max-output-length (count output)))+ (is (= 1 (count (re-seq #"Skill IDs you can load: read-resource\." output)))))))++(deftest ^:parallel load-skill-logs-refused-ids-test+ (testing "a missing or hidden skill leaves an info line, so operators can see a model guessing skill IDs"+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.skills :info]]+ (load-skills [:read-resource] ["does-not-exist" "edit-chart" "read-resource"])+ (is (= ["Unknown skill requested by load_skill"+ "Skill not available to current request {:id edit-chart}"]+ (map :message (logs)))))))++(deftest ^:parallel load-skill-needs-an-id-test+ (testing "an empty `ids` is an argument error that the model can correct"+ (is (= {:output "Invalid tool arguments: `ids` must be an array of at least one skill id; received an array."+ :error {:class :validation :code :invalid-arguments}}+ (load-skills [:read-resource] [])))))(deftest ^:parallel load-skill-unexpected-error-test- (testing "an unexpected error propagates to the agent loop"+ (testing "an unexpected error fails the whole call"(mt/with-dynamic-fn-redefs [skills/get-skill-by-id-string (fn [_] (throw (ex-info "boom" {})))]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"- (tools.skills/load-skill-tool {:ids ["construct-notebook-query-core"]}))))))+ (is (= {:class :unrecoverable :code :internal}+ (:error (load-skills [:construct-notebook-query-core] ["construct-notebook-query-core"])))))))diff --git a/test/metabase/metabot/tools/timelines_test.clj b/test/metabase/metabot/tools/timelines_test.cljindex 57dbbbf2f31..1b3011a09c0 100644--- a/test/metabase/metabot/tools/timelines_test.clj+++ b/test/metabase/metabot/tools/timelines_test.clj@@ -8,30 +8,28 @@[metabase.metabot.tools.timelines :as tools.timelines][metabase.test :as mt]))+(defn- list-timelines+ "Call the `list_timelines` tool and return what the model reads."+ []+ (:output (tools.core/handle tools.timelines/list-timelines-tool {} {})))+(deftest list-timelines-tool-test(mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Collection {coll-id :id} {:name "Timeline Coll"}:model/Timeline {tl-id :id} {:name "Product Launches":description "Key product launch dates":collection_id coll-id}]- (testing "returns timelines with id, name, description"- (let [result (tools.timelines/list-timelines-tool {})- items (:structured_output result)- matching (filter #(= tl-id (:id %)) items)]- (is (seq matching) "expected the test timeline to appear")- (is (= {:id tl-id :name "Product Launches" :description "Key product launch dates"}- (first matching)))))- (testing "output contains XML-formatted text"- (let [{:keys [output]} (tools.timelines/list-timelines-tool {})]- (is (re-find #"<timelines>" output))- (is (re-find #"Product Launches" output))))+ (testing "each timeline reaches the model as XML, with its id, name and description"+ (let [output (list-timelines)]+ (is (str/starts-with? output "<timelines>\n"))+ (is (str/includes? output (str "<timeline id=\"" tl-id "\" name=\"Product Launches\">"+ "Key product launch dates</timeline>")))+ (is (str/ends-with? output "\n</timelines>"))))(testing "archived timelines are excluded by default"(mt/with-temp [:model/Timeline _ {:name "Archived TL":collection_id coll-id:archived true}]- (let [items (:structured_output (tools.timelines/list-timelines-tool {}))- names (set (map :name items))]- (is (not (contains? names "Archived TL")))))))))+ (is (not (str/includes? (list-timelines) "Archived TL"))))))))(defn- details"Call the converted `get_timeline_details` tool for `timeline-id`."@@ -92,7 +90,7 @@:timestamp #t "2025-06-01T00:00:00Z":icon "star"}](testing "list output escapes XML-special characters in names and descriptions"- (let [{:keys [output]} (tools.timelines/list-timelines-tool {})]+ (let [output (list-timelines)](is (re-find #"name=\"Q&A <"Launches">\"" output))(is (re-find #"Tags: <b> & "quotes"" output))))(testing "details output escapes XML-special characters in timeline and event fields"diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.cljindex 6da26a4a12a..55429548045 100644--- a/test/metabase/metabot/tools_test.clj+++ b/test/metabase/metabot/tools_test.clj@@ -24,6 +24,7 @@[metabase.search.entity-retrieval.core :as entity-retrieval][metabase.test :as mt][metabase.util.json :as json]+ [metabase.util.malli :as mu][toucan2.core :as t2]))(deftest all-tools-test@@ -886,6 +887,13 @@{:query {:lib/type "mbql/query" :stages [stage]}:title "Orders" :description "Orders by count."})))))))))+(mu/defn ^{:tool-name "unconverted"}+ unconverted-tool+ "A tool declared through its var's metadata. The real ones are converted one at a time, so the tests+ keep their own."+ [{:keys [text]} :- [:map {:closed true} [:text :string]]]+ text)+(deftest ->entries-test(testing "an entry carries what the adapters and the system message read"(let [entries (agent-tools/->entries [#'agent-tools/create-sql-query-tool@@ -900,15 +908,15 @@(is (some? (:schema entry)))(is (fn? (:fn entry))))))))(testing "an unconverted tool's schema comes from its var metadata"- (let [entries (agent-tools/->entries [#'agent-tools/list-timelines-tool] (atom {}) nil :nlq)- [_:=> [_:cat declared] _out] (:schema (get entries "list_timelines"))- [_:=> [_:cat original] _out] (:schema (meta #'agent-tools/list-timelines-tool))]+ (let [entries (agent-tools/->entries [#'unconverted-tool] (atom {}) nil :nlq)+ [_:=> [_:cat declared] _out] (:schema (get entries "unconverted"))+ [_:=> [_:cat original] _out] (:schema (meta #'unconverted-tool))](is (= original declared))))(testing "a converted tool and an unconverted one sit side by side"- (let [entries (agent-tools/->entries [#'agent-tools/list-timelines-tool ; mu/defn var+ (let [entries (agent-tools/->entries [#'unconverted-tool ; mu/defn var#'agent-tools/get-timeline-details-tool] ; record(atom {}) nil :nlq)]- (is (= #{"list_timelines" "get_timeline_details"} (set (keys entries))))+ (is (= #{"unconverted" "get_timeline_details"} (set (keys entries))))(is (every? fn? (map :fn (vals entries))))))(testing "the entry's :fn goes through the runtime, so it reports rather than throws"(let [entries (agent-tools/->entries [#'agent-tools/search-tool] (atom {}) nil :nlq)